audio: implement random SFX output

Cargo.lock

@@ -492,6 +492,17 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
+[[package]]
+name = "chacha20"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.44"
@@ -693,6 +704,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -1198,6 +1218,7 @@ dependencies = [
  "minify",
  "musicbrainz_rs",
  "oximedia-metering",
+ "rand 0.10.1",
  "ratatui",
  "rc-writer",
  "rdf-types",
@@ -1210,6 +1231,7 @@ dependencies = [
  "sqlite",
  "static-iref",
  "static_cell",
+ "symphonia",
  "tempfile",
  "textwrap",
  "throbber-widgets-tui",
@@ -1252,6 +1274,12 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "extended"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
+
 [[package]]
 name = "eyre"
 version = "0.6.12"
@@ -1524,6 +1552,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "r-efi 6.0.0",
+ "rand_core 0.10.1",
  "wasip2",
  "wasip3",
 ]
@@ -3418,6 +3447,17 @@ dependencies = [
  "rand_core 0.9.5",
 ]
 
+[[package]]
+name = "rand"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
+dependencies = [
+ "chacha20",
+ "getrandom 0.4.2",
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.9.0"
@@ -3443,6 +3483,12 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+
 [[package]]
 name = "rand_xoshiro"
 version = "0.6.0"
@@ -3654,6 +3700,12 @@ dependencies = [
  "regex-syntax",
 ]
 
+[[package]]
+name = "regex-lite"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
+
 [[package]]
 name = "regex-syntax"
 version = "0.8.10"
@@ -4165,7 +4217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
  "digest",
 ]
 
@@ -4526,6 +4578,179 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
+[[package]]
+name = "symphonia"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a"
+dependencies = [
+ "lazy_static",
+ "symphonia-bundle-flac",
+ "symphonia-bundle-mp3",
+ "symphonia-codec-aac",
+ "symphonia-codec-adpcm",
+ "symphonia-codec-alac",
+ "symphonia-codec-pcm",
+ "symphonia-codec-vorbis",
+ "symphonia-core",
+ "symphonia-format-mkv",
+ "symphonia-format-ogg",
+ "symphonia-format-riff",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-bundle-flac"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03"
+dependencies = [
+ "log",
+ "symphonia-common",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-bundle-mp3"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-aac"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-common",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-adpcm"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebbdfd76d6cc5a601c6292a44357c5b7c82f2cd7cdc0f171421f5c5cff0ea1f"
+dependencies = [
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-alac"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8"
+dependencies = [
+ "log",
+ "symphonia-common",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-pcm"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328"
+dependencies = [
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-vorbis"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de"
+dependencies = [
+ "log",
+ "symphonia-common",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-common"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-core"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1"
+dependencies = [
+ "bitflags 2.11.1",
+ "bytemuck",
+ "lazy_static",
+ "log",
+ "num-complex",
+ "rustfft",
+ "smallvec",
+]
+
+[[package]]
+name = "symphonia-format-mkv"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-common",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-format-ogg"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433"
+dependencies = [
+ "log",
+ "symphonia-common",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-format-riff"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0"
+dependencies = [
+ "extended",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-metadata"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681"
+dependencies = [
+ "lazy_static",
+ "log",
+ "regex-lite",
+ "smallvec",
+ "symphonia-core",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.109"

Cargo.toml

@@ -22,6 +22,7 @@ log = "0.4.32"
 minify = "1.3.0"
 musicbrainz_rs = { version = "0.13.0", features = ["async"] }
 oximedia-metering = "0.1.7"
+rand = "0.10.1"
 ratatui = "0.30.0"
 rc-writer = "1.1.10"
 rdf-types = "0.22.5"
@@ -34,6 +35,7 @@ serde_json = "1.0.150"
 sqlite = "0.37.0"
 static-iref = "3.0.0"
 static_cell = "2.1.1"
+symphonia = { version = "0.6.0", features = ["all-codecs"] }
 tempfile = "3.27.0"
 textwrap = "0.16.2"
 throbber-widgets-tui = "0.11.0"

src/audio.rs

@@ -85,10 +85,7 @@ impl Display for Role {
 
 impl Role {
     fn is_input(&self) -> bool {
-        match self {
+        matches!(self, Role::Mic)
-            Role::Mic => true,
-            _ => false
-        }
     }
 }
 
@@ -126,7 +123,7 @@ impl AudioSource {
 
     fn process(&mut self, scope: &ProcessScope) -> Result<Option<f64>, AudioError> {
         if self.port.connected_count()? > 0 {
-            let buf: Vec<_> = self.port.as_slice(scope).iter().copied().collect();
+            let buf: Vec<_> = self.port.as_slice(scope).to_vec();
             self.meter.process_interleaved(&buf);
             self.sample_sink.blocking_send(buf)?;
 
@@ -159,7 +156,11 @@ impl AudioSink {
     }
 
     fn process(&mut self, scope: &ProcessScope) -> Result<(), AudioError> {
-        let mut next_outbuf = self.sample_src.try_recv()?;
+        let mut next_outbuf = match self.sample_src.try_recv() {
+            Ok(buf) => buf,
+            Err(tokio::sync::mpsc::error::TryRecvError::Empty) => return Ok(()),
+            Err(err) => return Err(err.into())
+        };
         self.output_buf.append(&mut next_outbuf);
 
         if self.port.connected_count()? > 0 && !self.output_buf.is_empty() {
@@ -167,9 +168,7 @@ impl AudioSink {
             let mut next_segment: Vec<f32> = self.output_buf.drain(0..(outbuf.len()).min(self.output_buf.len())).collect();
             let underrun = outbuf.len() - next_segment.len();
             if underrun > 0 {
-                for _ in 0..underrun {
+                next_segment.extend(std::iter::repeat_n(0., underrun));
-                    next_segment.push(0.);
-                }
             }
 
             outbuf.copy_from_slice(&next_segment);
@@ -228,7 +227,7 @@ impl NotificationHandler for Notify {
         }).next();
 
         if let Some((role, Ok(target_port))) = port_match {
-            let cfg_slot = self.config.connections.entry(*role).or_insert_with(|| Default::default());
+            let cfg_slot = self.config.connections.entry(*role).or_default();
             
             if are_connected {
                 log::info!("{} connected to {}", role, target_port);
@@ -278,7 +277,7 @@ pub async fn start_audio_input() -> (AudioInputControl, AudioInStream, AudioOutS
                     } else {
                         (local_port, &peer)
                     };
-                    if let Err(err) = client.connect_ports(&src, &dst) {
+                    if let Err(err) = client.connect_ports(src, dst) {
                         log::error!("Failed to reconnect {} to {}: {:?}", role, peer_name, err);
                     } else {
                         log::info!("Reconnected {} to {}", role, peer_name);

src/main.rs

@@ -10,7 +10,7 @@ use futures::StreamExt;
 
 use ratatui::prelude::*;
 
-use crate::{artifacts::archive::Archive, audio::start_audio_input, scene::{StageDirection, conversation::ConversationEntry}, tts::start_tts, ui::Ui};
+use crate::{artifacts::archive::Archive, audio::start_audio_input, scene::{StageDirection, conversation::ConversationEntry}, sfx::start_sfx, tts::start_tts, ui::Ui};
 
 mod scene;
 mod events;
@@ -21,6 +21,7 @@ mod audio;
 mod artifacts;
 mod ui;
 mod widgets;
+mod sfx;
 
 // TODO: We should be able to delete entries from the conversation, or at least go back and edit something I said.
 // TODO: I want a "mark" command or keyboard shortcut, that inserts a marker into the log, so I know where to come back for the next speaking segment.
@@ -131,11 +132,14 @@ async fn main() {
         SaveData::default()
     };
 
-    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src).await;
+    let (audio_ctrl, mic_stream, tts_output, sfx_output) = start_audio_input().await;
-    let (audio_ctrl, mic_stream, tts_output, _sfx_output) = start_audio_input().await;
     let tts_ctrl = start_tts(tts_output).await;
+    let mut sfx_ctrl = start_sfx(sfx_output).await;
+    sfx_ctrl.play_ambient().await.unwrap();
     let transcription_ctrl = transcription::start_transcription(mic_stream).await;
 
+    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl).await;
+
     let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl);
 
     let mut events = EventStream::new();

src/prediction/mod.rs

@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::{Serializer, Value, ser::CompactFormatter};
 use tokio::sync::{Mutex, mpsc::{self, UnboundedReceiver, UnboundedSender}};
 
-use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}};
+use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl};
 use tokio_stream::StreamExt;
 
 pub mod character;
@@ -68,7 +68,7 @@ impl SessionControl {
     }
 
     pub async fn changed(&mut self) -> SessionUpdate {
-        self.event_src.recv().await.unwrap()
+        self.event_src.recv().await.expect("Session closed")
     }
 }
 
@@ -84,7 +84,9 @@ struct Conversation {
     computer_todo: Arc<Mutex<HashMap<String, bool>>>,
     archive: Arc<Mutex<Archive>>,
     current_playlist: Vec<Track>,
-    sys_log_messages: UnboundedReceiver<ConversationEntry>
+    sys_log_messages: UnboundedReceiver<ConversationEntry>,
+
+    sfx: SfxControl
 }
 
 impl Conversation {
@@ -160,6 +162,7 @@ impl Conversation {
                 self.event_sink.send(SessionUpdate::Responses(next_options)).unwrap();
             },
             Speaker::ShipComputer => {
+                self.sfx.play_ambient().await.unwrap();
                 let response: ComputerResponse = serde_json::from_value(value).unwrap();
                 self.insert(ConversationEntry::Spoken(Speaker::ShipComputer, response.message)).await;
                 if response.finished.unwrap_or_default() {
@@ -279,7 +282,7 @@ impl Conversation {
     }
 }
 
-pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::sync::mpsc::UnboundedReceiver<ConversationEntry>) -> SessionControl {
+pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::sync::mpsc::UnboundedReceiver<ConversationEntry>, sfx: SfxControl ) -> SessionControl {
     let (input_sink, input_src) = tokio::sync::mpsc::unbounded_channel();
     let (event_sink, event_src) = tokio::sync::mpsc::unbounded_channel();
 
@@ -330,7 +333,8 @@ pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::syn
         archive,
         current_playlist: vec![],
         sys_log_messages,
-        computer_todo: shared_todo
+        computer_todo: shared_todo,
+        sfx
     };
     
     tokio::spawn(async move {

src/sfx.rs

@@ -0,0 +1,81 @@
+use rand::seq::IteratorRandom;
+use symphonia::core::{formats::{TrackType, probe::Hint}, io::MediaSourceStream};
+
+use crate::audio::AudioOutStream;
+
+#[derive(Debug)]
+pub enum SfxRequest {
+    RandomAmbient
+}
+
+#[derive(Debug)]
+pub struct SfxControl {
+    sink: tokio::sync::mpsc::Sender<SfxRequest>
+}
+
+impl SfxControl {
+    pub async fn play_ambient(&mut self) -> Result<(), tokio::sync::mpsc::error::SendError<SfxRequest>> {
+        self.sink.send(SfxRequest::RandomAmbient).await
+    }
+}
+
+pub async fn start_sfx(audio_sink: AudioOutStream) -> SfxControl {
+    let (event_sink, mut event_src) = tokio::sync::mpsc::channel(32);
+    tokio::spawn(async move {
+        let sfx_dir = std::path::Path::new("./sfx");
+
+        loop {
+            while let Some(event) = event_src.recv().await {
+                match event {
+                    SfxRequest::RandomAmbient => {
+                        let avail_files = std::fs::read_dir(sfx_dir).unwrap();
+                        let chosen_file = avail_files.choose(&mut rand::rng()).unwrap().unwrap();
+                        log::debug!("Queuing ambient sound playback with {:?}", chosen_file);
+                        let sfx_fd = std::fs::File::open(chosen_file.path()).unwrap();
+                        let mss = MediaSourceStream::new(Box::new(sfx_fd), Default::default());
+                        let meta_opts = Default::default();
+                        let fmt_opts = Default::default();
+                        let mut hint = Hint::new();
+                        hint.with_extension(".mp3");
+                        let mut format = symphonia::default::get_probe()
+                            .probe(&hint, mss, fmt_opts, meta_opts)
+                            .expect("Unsupported audio format");
+                        let track = format.default_track(TrackType::Audio).expect("No audio track");
+                        let dec_opts = Default::default();
+                        let mut decoder = symphonia::default::get_codecs()
+                            .make_audio_decoder(
+                                track.codec_params.as_ref().expect("codec params missing").audio().unwrap(),
+                                &dec_opts
+                            ).expect("Unsupported audio codec");
+                        log::debug!("Starting stream");
+                        loop {
+                            let packet = match format.next_packet() {
+                                Ok(Some(packet)) => packet,
+                                Ok(None) => break,
+                                Err(err) => panic!()
+                            };
+
+                            match decoder.decode_ref(&packet.as_packet_ref()) {
+                                Ok(samples) => {
+                                    let mut channel_bufs: Vec<f32> = vec![];
+                                    samples.copy_to_vec_interleaved(&mut channel_bufs);
+                                    let audio_out_buf: Vec<f32> = channel_bufs.windows(samples.byte_len_per_plane()).map(|channels| {
+                                        let total_volume = channels.iter().cloned().reduce(|a, b| a + b).unwrap_or_default();
+                                        total_volume / (channels.len() as f32)
+                                    }).collect();
+                                    audio_sink.sink.send(audio_out_buf).await.unwrap();
+                                },
+                                Err(err) => panic!()
+                            }
+                        }
+                        log::debug!("Playback complete");
+                    }
+                }
+            }
+        }
+    });
+
+    SfxControl {
+        sink: event_sink
+    }
+}