From 74a823d1c21eb0522a2c4f53c6c8e655b6085d3c Mon Sep 17 00:00:00 2001 From: Victoria Fischer Date: Mon, 22 Jun 2026 13:54:42 +0200 Subject: [PATCH] audio: implement random SFX output --- Cargo.lock | 227 +++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + src/audio.rs | 21 ++-- src/main.rs | 10 +- src/prediction/mod.rs | 14 ++- src/sfx.rs | 81 +++++++++++++++ 6 files changed, 335 insertions(+), 20 deletions(-) create mode 100644 src/sfx.rs diff --git a/Cargo.lock b/Cargo.lock index 117dc74..cb754c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -492,6 +492,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -693,6 +704,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1198,6 +1218,7 @@ dependencies = [ "minify", "musicbrainz_rs", "oximedia-metering", + "rand 0.10.1", "ratatui", "rc-writer", "rdf-types", @@ -1210,6 +1231,7 @@ dependencies = [ "sqlite", "static-iref", "static_cell", + "symphonia", "tempfile", "textwrap", "throbber-widgets-tui", @@ -1252,6 +1274,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "eyre" version = "0.6.12" @@ -1524,6 +1552,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3418,6 +3447,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -3443,6 +3483,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -3654,6 +3700,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -4165,7 +4217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4526,6 +4578,179 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebbdfd76d6cc5a601c6292a44357c5b7c82f2cd7cdc0f171421f5c5cff0ea1f" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1" +dependencies = [ + "bitflags 2.11.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "rustfft", + "smallvec", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index a1a488e..a280d86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ log = "0.4.32" minify = "1.3.0" musicbrainz_rs = { version = "0.13.0", features = ["async"] } oximedia-metering = "0.1.7" +rand = "0.10.1" ratatui = "0.30.0" rc-writer = "1.1.10" rdf-types = "0.22.5" @@ -34,6 +35,7 @@ serde_json = "1.0.150" sqlite = "0.37.0" static-iref = "3.0.0" static_cell = "2.1.1" +symphonia = { version = "0.6.0", features = ["all-codecs"] } tempfile = "3.27.0" textwrap = "0.16.2" throbber-widgets-tui = "0.11.0" diff --git a/src/audio.rs b/src/audio.rs index 1b889e3..1620d2b 100644 --- a/src/audio.rs +++ b/src/audio.rs @@ -85,10 +85,7 @@ impl Display for Role { impl Role { fn is_input(&self) -> bool { - match self { - Role::Mic => true, - _ => false - } + matches!(self, Role::Mic) } } @@ -126,7 +123,7 @@ impl AudioSource { fn process(&mut self, scope: &ProcessScope) -> Result, AudioError> { if self.port.connected_count()? > 0 { - let buf: Vec<_> = self.port.as_slice(scope).iter().copied().collect(); + let buf: Vec<_> = self.port.as_slice(scope).to_vec(); self.meter.process_interleaved(&buf); self.sample_sink.blocking_send(buf)?; @@ -159,7 +156,11 @@ impl AudioSink { } fn process(&mut self, scope: &ProcessScope) -> Result<(), AudioError> { - let mut next_outbuf = self.sample_src.try_recv()?; + let mut next_outbuf = match self.sample_src.try_recv() { + Ok(buf) => buf, + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => return Ok(()), + Err(err) => return Err(err.into()) + }; self.output_buf.append(&mut next_outbuf); if self.port.connected_count()? > 0 && !self.output_buf.is_empty() { @@ -167,9 +168,7 @@ impl AudioSink { let mut next_segment: Vec = self.output_buf.drain(0..(outbuf.len()).min(self.output_buf.len())).collect(); let underrun = outbuf.len() - next_segment.len(); if underrun > 0 { - for _ in 0..underrun { - next_segment.push(0.); - } + next_segment.extend(std::iter::repeat_n(0., underrun)); } outbuf.copy_from_slice(&next_segment); @@ -228,7 +227,7 @@ impl NotificationHandler for Notify { }).next(); if let Some((role, Ok(target_port))) = port_match { - let cfg_slot = self.config.connections.entry(*role).or_insert_with(|| Default::default()); + let cfg_slot = self.config.connections.entry(*role).or_default(); if are_connected { log::info!("{} connected to {}", role, target_port); @@ -278,7 +277,7 @@ pub async fn start_audio_input() -> (AudioInputControl, AudioInStream, AudioOutS } else { (local_port, &peer) }; - if let Err(err) = client.connect_ports(&src, &dst) { + if let Err(err) = client.connect_ports(src, dst) { log::error!("Failed to reconnect {} to {}: {:?}", role, peer_name, err); } else { log::info!("Reconnected {} to {}", role, peer_name); diff --git a/src/main.rs b/src/main.rs index b415280..de402c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ use futures::StreamExt; use ratatui::prelude::*; -use crate::{artifacts::archive::Archive, audio::start_audio_input, scene::{StageDirection, conversation::ConversationEntry}, tts::start_tts, ui::Ui}; +use crate::{artifacts::archive::Archive, audio::start_audio_input, scene::{StageDirection, conversation::ConversationEntry}, sfx::start_sfx, tts::start_tts, ui::Ui}; mod scene; mod events; @@ -21,6 +21,7 @@ mod audio; mod artifacts; mod ui; mod widgets; +mod sfx; // TODO: We should be able to delete entries from the conversation, or at least go back and edit something I said. // TODO: I want a "mark" command or keyboard shortcut, that inserts a marker into the log, so I know where to come back for the next speaking segment. @@ -131,11 +132,14 @@ async fn main() { SaveData::default() }; - let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src).await; - let (audio_ctrl, mic_stream, tts_output, _sfx_output) = start_audio_input().await; + let (audio_ctrl, mic_stream, tts_output, sfx_output) = start_audio_input().await; let tts_ctrl = start_tts(tts_output).await; + let mut sfx_ctrl = start_sfx(sfx_output).await; + sfx_ctrl.play_ambient().await.unwrap(); let transcription_ctrl = transcription::start_transcription(mic_stream).await; + let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl).await; + let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl); let mut events = EventStream::new(); diff --git a/src/prediction/mod.rs b/src/prediction/mod.rs index 4514321..fcd981a 100644 --- a/src/prediction/mod.rs +++ b/src/prediction/mod.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Serializer, Value, ser::CompactFormatter}; use tokio::sync::{Mutex, mpsc::{self, UnboundedReceiver, UnboundedSender}}; -use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}}; +use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl}; use tokio_stream::StreamExt; pub mod character; @@ -68,7 +68,7 @@ impl SessionControl { } pub async fn changed(&mut self) -> SessionUpdate { - self.event_src.recv().await.unwrap() + self.event_src.recv().await.expect("Session closed") } } @@ -84,7 +84,9 @@ struct Conversation { computer_todo: Arc>>, archive: Arc>, current_playlist: Vec, - sys_log_messages: UnboundedReceiver + sys_log_messages: UnboundedReceiver, + + sfx: SfxControl } impl Conversation { @@ -160,6 +162,7 @@ impl Conversation { self.event_sink.send(SessionUpdate::Responses(next_options)).unwrap(); }, Speaker::ShipComputer => { + self.sfx.play_ambient().await.unwrap(); let response: ComputerResponse = serde_json::from_value(value).unwrap(); self.insert(ConversationEntry::Spoken(Speaker::ShipComputer, response.message)).await; if response.finished.unwrap_or_default() { @@ -279,7 +282,7 @@ impl Conversation { } } -pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::sync::mpsc::UnboundedReceiver) -> SessionControl { +pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::sync::mpsc::UnboundedReceiver, sfx: SfxControl ) -> SessionControl { let (input_sink, input_src) = tokio::sync::mpsc::unbounded_channel(); let (event_sink, event_src) = tokio::sync::mpsc::unbounded_channel(); @@ -330,7 +333,8 @@ pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::syn archive, current_playlist: vec![], sys_log_messages, - computer_todo: shared_todo + computer_todo: shared_todo, + sfx }; tokio::spawn(async move { diff --git a/src/sfx.rs b/src/sfx.rs new file mode 100644 index 0000000..84cc12c --- /dev/null +++ b/src/sfx.rs @@ -0,0 +1,81 @@ +use rand::seq::IteratorRandom; +use symphonia::core::{formats::{TrackType, probe::Hint}, io::MediaSourceStream}; + +use crate::audio::AudioOutStream; + +#[derive(Debug)] +pub enum SfxRequest { + RandomAmbient +} + +#[derive(Debug)] +pub struct SfxControl { + sink: tokio::sync::mpsc::Sender +} + +impl SfxControl { + pub async fn play_ambient(&mut self) -> Result<(), tokio::sync::mpsc::error::SendError> { + self.sink.send(SfxRequest::RandomAmbient).await + } +} + +pub async fn start_sfx(audio_sink: AudioOutStream) -> SfxControl { + let (event_sink, mut event_src) = tokio::sync::mpsc::channel(32); + tokio::spawn(async move { + let sfx_dir = std::path::Path::new("./sfx"); + + loop { + while let Some(event) = event_src.recv().await { + match event { + SfxRequest::RandomAmbient => { + let avail_files = std::fs::read_dir(sfx_dir).unwrap(); + let chosen_file = avail_files.choose(&mut rand::rng()).unwrap().unwrap(); + log::debug!("Queuing ambient sound playback with {:?}", chosen_file); + let sfx_fd = std::fs::File::open(chosen_file.path()).unwrap(); + let mss = MediaSourceStream::new(Box::new(sfx_fd), Default::default()); + let meta_opts = Default::default(); + let fmt_opts = Default::default(); + let mut hint = Hint::new(); + hint.with_extension(".mp3"); + let mut format = symphonia::default::get_probe() + .probe(&hint, mss, fmt_opts, meta_opts) + .expect("Unsupported audio format"); + let track = format.default_track(TrackType::Audio).expect("No audio track"); + let dec_opts = Default::default(); + let mut decoder = symphonia::default::get_codecs() + .make_audio_decoder( + track.codec_params.as_ref().expect("codec params missing").audio().unwrap(), + &dec_opts + ).expect("Unsupported audio codec"); + log::debug!("Starting stream"); + loop { + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + Err(err) => panic!() + }; + + match decoder.decode_ref(&packet.as_packet_ref()) { + Ok(samples) => { + let mut channel_bufs: Vec = vec![]; + samples.copy_to_vec_interleaved(&mut channel_bufs); + let audio_out_buf: Vec = channel_bufs.windows(samples.byte_len_per_plane()).map(|channels| { + let total_volume = channels.iter().cloned().reduce(|a, b| a + b).unwrap_or_default(); + total_volume / (channels.len() as f32) + }).collect(); + audio_sink.sink.send(audio_out_buf).await.unwrap(); + }, + Err(err) => panic!() + } + } + log::debug!("Playback complete"); + } + } + } + } + }); + + SfxControl { + sink: event_sink + } +} \ No newline at end of file