ui: add some sfx integration to the UI, fix a json crash

src/main.rs

@@ -134,13 +134,12 @@ async fn main() {
 
     let (audio_ctrl, mic_stream, tts_output, sfx_output) = start_audio_input().await;
     let tts_ctrl = start_tts(tts_output).await;
-    let mut sfx_ctrl = start_sfx(sfx_output).await;
-    sfx_ctrl.play_ambient().await.unwrap();
+    let sfx_ctrl = start_sfx(sfx_output).await;
     let transcription_ctrl = transcription::start_transcription(mic_stream).await;
 
-    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl).await;
+    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl.clone()).await;
 
-    let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl);
+    let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl, sfx_ctrl);
 
     let mut events = EventStream::new();
 

src/prediction/mod.rs

@@ -99,7 +99,11 @@ impl Conversation {
         self.backlog.push(entry.clone());
         self.event_sink.send(SessionUpdate::Conversation(self.backlog.clone())).unwrap();
         match entry {
-            ConversationEntry::Spoken(_, _) => {
+            ConversationEntry::Spoken(speaker, _) => {
+                match speaker {
+                    Speaker::User => (),
+                    _ => self.sfx.play_ambient().await.unwrap(),
+                }
                 if let Ok(next_msg) = TryInto::<ChatCompletionRequestMessage>::try_into(entry) {
                     self.send_to(Speaker::Eva, next_msg.clone()).await;
                     let cxt = self.context_for_speaker(Speaker::Eva).await;
@@ -158,11 +162,11 @@ impl Conversation {
     async fn process_dialog(&mut self, speaker: Speaker, value: Value) {
         match speaker {
             Speaker::Eva => {
-                let next_options = serde_json::from_value(value).unwrap();
+                // TODO: Handle crash from bad json
+                let next_options = serde_json::from_value(value).unwrap_or_default();
                 self.event_sink.send(SessionUpdate::Responses(next_options)).unwrap();
             },
             Speaker::ShipComputer => {
-                self.sfx.play_ambient().await.unwrap();
                 let response: ComputerResponse = serde_json::from_value(value).unwrap();
                 self.insert(ConversationEntry::Spoken(Speaker::ShipComputer, response.message)).await;
                 if response.finished.unwrap_or_default() {
@@ -326,8 +330,8 @@ pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::syn
         ]),
         event_sink,
         input_src,
+        eva_backlog: save_data.messages,
         backlog,
-        eva_backlog: Default::default(),
         tokens_consumed: save_data.tokens_consumed,
         direction: save_data.direction,
         archive,

src/ui.rs

@@ -6,7 +6,7 @@ use tokio::time::Instant;
 use tui_input::{Input, backend::crossterm::EventHandler};
 use tui_skeleton::{AnimationMode, Block, Constraint, SkeletonText};
 
-use crate::{audio::AudioInputControl, prediction::{GeneratedResponses, PredictionAction, SessionControl, SessionUpdate}, scene::{Scene, conversation::{ConversationEntry, Speaker}}, transcription::TranscriptionControl, tts::TtsControl};
+use crate::{audio::AudioInputControl, prediction::{GeneratedResponses, PredictionAction, SessionControl, SessionUpdate}, scene::{Scene, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl, transcription::TranscriptionControl, tts::TtsControl};
 use crate::widgets::*;
 
 #[derive(Debug)]
@@ -30,7 +30,8 @@ pub struct Ui {
     audio: AudioInputControl,
     tts: TtsControl,
     predictions: SessionControl,
-    conversation: Vec<ConversationEntry>
+    conversation: Vec<ConversationEntry>,
+    sfx: SfxControl
 }
 
 #[derive(Debug)]
@@ -40,7 +41,7 @@ enum FocusState {
 }
 
 impl Ui {
-    pub fn new(predictions: SessionControl, audio: AudioInputControl, transcription: TranscriptionControl, tts: TtsControl) -> Self {
+    pub fn new(predictions: SessionControl, audio: AudioInputControl, transcription: TranscriptionControl, tts: TtsControl, sfx: SfxControl) -> Self {
         Self {
             scene: Default::default(),
             reply_state: Default::default(), 
@@ -58,7 +59,8 @@ impl Ui {
             predictions,
             last_tick: Instant::now(),
             conversation: vec![],
-            reply_options: Default::default()
+            reply_options: Default::default(),
+            sfx
         }
     }
 
@@ -193,6 +195,9 @@ impl Ui {
             "/computer" => {
                 self.predictions.insert(PredictionAction::ComputerCommand(arg.to_string())).await;
             },
+            "/ambient" => {
+                self.sfx.play_ambient().await.unwrap()
+            },
             _ => {
                 log::error!("Unknown command. Available commands: /episode [number], /narrative [text], /event [text], /computer [text], /timer [minutes]");
             }
@@ -220,6 +225,7 @@ impl Ui {
                             let row_num = self.conversation_state.selected().unwrap();
                             if let ConversationEntry::Spoken(Speaker::Eva, text) = &self.conversation[self.conversation.len() - 1 - row_num] {
                                 self.tts.speak(text.clone()).await.unwrap();
+                                self.sfx.play_ambient().await.unwrap();
                                 self.focus_state = FocusState::UserInput;
                                 self.conversation_state.select(None);
                                 self.reply_state.select_first();