gitignore: add audio.json and .env

.gitignore

@@ -1,2 +1,4 @@
 /target
 save.json
+.env
+audio.json

Cargo.lock

@@ -260,6 +260,16 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "better-panic"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fa9e1d11a268684cbd90ed36370d7577afb6c62d912ddff5c15fc34343e5036"
+dependencies = [
+ "backtrace",
+ "console",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.3"
@@ -493,6 +503,18 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "contextual"
 version = "0.1.6"
@@ -819,6 +841,12 @@ dependencies = [
  "litrs",
 ]
 
+[[package]]
+name = "dotenv"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
+
 [[package]]
 name = "dtoa"
 version = "1.0.11"
@@ -882,6 +910,12 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
 
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.35"
@@ -955,9 +989,11 @@ version = "0.1.0"
 dependencies = [
  "async-openai",
  "bandcamp",
+ "better-panic",
  "chrono",
  "color-eyre",
  "crossterm",
+ "dotenv",
  "futures",
  "futures-timer",
  "hound",
@@ -978,9 +1014,11 @@ dependencies = [
  "sqlite",
  "static-iref",
  "tempfile",
+ "textwrap",
  "throbber-widgets-tui",
  "tokio",
  "tui-input",
+ "tui-skeleton",
 ]
 
 [[package]]
@@ -3911,6 +3949,12 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
+[[package]]
+name = "smawk"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
+
 [[package]]
 name = "snafu"
 version = "0.8.9"
@@ -4275,6 +4319,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057"
+dependencies = [
+ "smawk",
+ "unicode-linebreak",
+ "unicode-width",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -4590,6 +4645,16 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "tui-skeleton"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d3eb5b0d4f9fc43565038283e53913bfd961a63f914d3352a71348c695e508e"
+dependencies = [
+ "ratatui-core",
+ "ratatui-widgets",
+]
+
 [[package]]
 name = "tungstenite"
 version = "0.28.0"
@@ -4627,6 +4692,12 @@ version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
 
+[[package]]
+name = "unicode-linebreak"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.13.2"
@@ -5140,6 +5211,15 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.60.2"

Cargo.toml

@@ -6,9 +6,11 @@ edition = "2024"
 [dependencies]
 async-openai = { version = "0.40.2", features = ["completions", "full"] }
 bandcamp = "0.3.4"
+better-panic = "0.3.0"
 chrono = { version = "0.4.44", features = ["serde"] }
 color-eyre = "0.6.5"
 crossterm = { version = "0.29.0", features = ["event-stream"] }
+dotenv = "0.15.0"
 futures = "0.3.32"
 futures-timer = "3.0.4"
 hound = "3.5.1"
@@ -29,6 +31,8 @@ serde_json = "1.0.150"
 sqlite = "0.37.0"
 static-iref = "3.0.0"
 tempfile = "3.27.0"
+textwrap = "0.16.2"
 throbber-widgets-tui = "0.11.0"
 tokio = { version = "1.52.3", features = ["full"] }
 tui-input = "0.15.3"
+tui-skeleton = "0.3.0"

build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("cargo::rerun-if-changed=src/system-prompt.txt");
+}

src/audio.rs

@@ -0,0 +1,200 @@
+use jack::{AudioIn, AudioOut, ClientOptions, NotificationHandler};
+use oximedia_metering::vu_meter::VuMeter;
+use serde::{Deserialize, Serialize};
+use tokio::sync::*;
+
+#[derive(Debug)]
+pub struct JackClientRef {
+    killswitch: Option<oneshot::Sender<()>>
+}
+
+impl Drop for JackClientRef {
+    fn drop(&mut self) {
+        self.killswitch.take().unwrap().send(()).unwrap();
+    }
+}
+
+#[derive(Debug)]
+pub struct AudioInputControl {
+    volume_src: watch::Receiver<f64>,
+    _jack_client: JackClientRef
+}
+
+impl AudioInputControl {
+    pub async fn next(&mut self) -> f64 {
+        self.volume_src.changed().await.unwrap();
+        *self.volume_src.borrow_and_update()
+    }
+}
+
+#[derive(Debug)]
+pub struct MicStream {
+    pub src: mpsc::Receiver<Vec<f32>>,
+    pub sample_rate: u32
+}
+
+#[derive(Debug)]
+pub struct TtsOutStream {
+    pub sink: mpsc::Sender<Vec<f32>>,
+    pub sample_rate: u32
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+struct AudioConfig {
+    mic_in_connections: Vec<String>,
+    tts_out_connections: Vec<String>
+}
+
+impl AudioConfig {
+    pub fn load() -> Self {
+        if let Ok(contents) = std::fs::read_to_string("audio.json") {
+            serde_json::from_str(contents.as_str()).unwrap()
+        } else {
+            Default::default()
+        }
+    }
+}
+
+#[derive(Debug)]
+struct Notify {
+    config: AudioConfig,
+    mic_port: jack::Port<jack::Unowned>,
+    tts_port: jack::Port<jack::Unowned>,
+    log: mpsc::Sender<String>
+}
+
+impl NotificationHandler for Notify {
+    fn ports_connected(
+        &mut self,
+        client: &jack::Client,
+        port_id_a: jack::PortId,
+        port_id_b: jack::PortId,
+        are_connected: bool,
+    ) {
+        let port_a = client.port_by_id(port_id_a).unwrap();
+        let port_b = client.port_by_id(port_id_b).unwrap();
+
+        let (stream_name, other_port, target_cfg) = if port_b == self.mic_port {
+            ("Microphone input", port_a, &mut self.config.mic_in_connections)
+        } else if port_a == self.tts_port {
+            ("TTS output", port_b, &mut self.config.tts_out_connections)
+        } else {
+            return;
+        };
+
+        if let Ok(port_name) = other_port.name() {
+            if are_connected {
+                self.log.blocking_send(format!("{} connected to {}", stream_name, port_name)).unwrap();
+                target_cfg.push(port_name);
+            } else {
+                self.log.blocking_send(format!("{} disconnected from {}", stream_name, port_name)).unwrap();
+                target_cfg.retain(|x| { x != &port_name} );
+            }
+
+            let save_data = serde_json::to_string_pretty(&self.config).unwrap();
+            std::fs::write("audio.json", save_data).unwrap();
+        }
+    }
+}
+
+pub async fn start_audio_input(messages: &mpsc::Sender<String>) -> (AudioInputControl, MicStream, TtsOutStream) {
+
+    let (exit_tx, exit_rx) = oneshot::channel();
+
+    let config = AudioConfig::load();
+
+    let (mic_audio_sink, mic_audio_src) = mpsc::channel(32);
+    let (tts_audio_sink, mut tts_audio_src) = mpsc::channel(32);
+    let (volume_sink, volume_src) = watch::channel(0.);
+
+    let (client, _status) = jack::Client::new("Eva-Cohost", ClientOptions::default() | ClientOptions::SESSION_ID).unwrap();
+    let mic_port = client.register_port("microphone-in", AudioIn::default()).unwrap();
+    let mut tts_port = client.register_port("tts-out", AudioOut::default()).unwrap();
+    let rate = client.sample_rate();
+
+    let mic_name = mic_port.name().unwrap();
+    let tts_name = tts_port.name().unwrap();
+
+    for port in &config.mic_in_connections {
+        if let Ok(_) = client.connect_ports_by_name(&port, &mic_name) {
+            messages.send(format!("Connected mic to {}", port)).await.unwrap();
+        } else {
+            messages.send(format!("Failed to reconnect mic to {}.", port)).await.unwrap();
+        }
+    }
+
+    for port in &config.tts_out_connections {
+        if let Ok(_) = client.connect_ports_by_name(&tts_name, &port) {
+            messages.send(format!("Connected TTS output to {}", port)).await.unwrap();
+        } else {
+            messages.send(format!("Failed to reconnect TTS output to {}.", port)).await.unwrap();
+        }
+    }
+
+    let notifier = Notify {
+        config,
+        mic_port: mic_port.clone_unowned(),
+        tts_port: tts_port.clone_unowned(),
+        log: messages.clone()
+    };
+
+    let mut meter = VuMeter::new(rate.into(), 1, None);
+    let mut tts_output_buf = vec![];
+    tts_output_buf.reserve(1024);
+
+    let handler = jack::contrib::ClosureProcessHandler::new(move |_client, scope| {
+        if mic_port.connected_count().unwrap() > 0 {
+            let buf: Vec<_> = mic_port.as_slice(scope).iter().copied().collect();
+            meter.process_interleaved(&buf);
+            mic_audio_sink.blocking_send(buf).unwrap();
+
+            volume_sink.send_if_modified(|v| {
+                let next_vu = meter.channel_vu(0).unwrap();
+                if *v != next_vu {
+                    *v = next_vu;
+                    true
+                } else {
+                    false
+                }
+            });
+        }
+
+        if let Ok(mut next_outbuf) = tts_audio_src.try_recv() {
+            tts_output_buf.append(&mut next_outbuf);
+        }   
+
+        if tts_port.connected_count().unwrap() > 0 && !tts_output_buf.is_empty() {
+            let outbuf = tts_port.as_mut_slice(scope);
+            let mut next_segment: Vec<f32> = tts_output_buf.drain(0..(outbuf.len()).min(tts_output_buf.len())).collect();
+            let underrun = outbuf.len() - next_segment.len();
+            if underrun > 0 {
+                for _ in 0..underrun {
+                    next_segment.push(0.);
+                }
+            }
+
+            outbuf.copy_from_slice(&next_segment);
+        }
+
+        jack::Control::Continue
+    });
+
+    tokio::spawn(async move {
+        let async_client = client.activate_async(notifier, handler).unwrap();
+
+        exit_rx.await.unwrap();
+
+        async_client.deactivate().unwrap();
+    });
+
+    (AudioInputControl {
+        volume_src,
+        _jack_client: JackClientRef { killswitch: Some(exit_tx) }
+    }, MicStream {
+        sample_rate: rate,
+        src: mic_audio_src
+    }, TtsOutStream {
+        sample_rate: rate,
+        sink: tts_audio_sink
+    })
+}

src/main.rs

@@ -1,11 +1,9 @@
 use async_openai::types::chat::ChatCompletionRequestMessage;
 use chrono::{DateTime, Duration, Utc};
 use futures_timer::Delay;
-use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
 
 use ratatui::{Frame, layout::{Constraint, Direction, Layout}, widgets::{Block, BorderType, Clear, Gauge, List, ListDirection, ListItem, ListState, Paragraph, Wrap}};
-use sqlite::OpenFlags;
 use throbber_widgets_tui::{Throbber, ThrobberState};
 use crossterm::{event::{self, EventStream, KeyCode, KeyModifiers}};
 use tokio::{sync::{mpsc, watch}, time::Instant};
@@ -13,16 +11,17 @@ use tui_input::{Input, backend::crossterm::EventHandler};
 use futures::{StreamExt, future::FutureExt};
 
 use ratatui::prelude::*;
+use tui_skeleton::{AnimationMode, SkeletonText};
 
-use crate::{events::AudioRecordRequest, prediction::{PossibleResponse}, scene::{ConversationEntry, PlaylistEntry, Scene, StageActions, StageDirection}, tts::start_tts};
+use crate::{audio::{AudioInputControl, start_audio_input}, prediction::{BandcampResult, PossibleResponse}, scene::{ConversationEntry, Scene, Scenery, StageActions, StageDirection}, transcription::TranscriptionControl, tts::{TtsControl, start_tts}};
 
 mod scene;
 mod events;
 mod transcription;
 mod tts;
 mod prediction;
+mod audio;
 
-// TODO: We should have a separate 'state.json' file, which remembers jack connections, and the world time for the show to end. Then we only update the 'time remaining' field in the scene and only deal with relative durations inside the scene data
 // TODO: We should be able to delete entries from the conversation, or at least go back and edit something I said.
 // TODO: I want a "mark" command or keyboard shortcut, that inserts a marker into the log, so I know where to come back for the next speaking segment.
 // TODO: If we insert text without speaking, this should be indicated visually somehow
@@ -32,6 +31,8 @@ mod prediction;
 // FIXME: It is unclear what would happen if we are live editing the save.json, have a typo, then reload. The file might get wiped without recovery.
 // TODO: Would be nice to have some SFX integrated, with bleeps and calculation nosies or something periodically
 
+// TODO: Should rewrite the entire prompt prediction loop, so the UI pushes modification events to the session and receives a new Scene from time to time whenever the prediction engine thinks it should regenerate one.
+
 /* Usage loop:
   - Prompt user to select one of:
     - Select response 1 (1)
@@ -54,8 +55,8 @@ impl<'a> Into<ListItem<'a>> for PossibleResponse {
     fn into(self) -> ListItem<'a> {
         if let Some(direction) = self.stage_direction {
             Line::from_iter([
-                Span::from(format!("({})", direction)).style(ratatui::style::Color::Yellow),
-                Span::from(" "),
+                //Span::from(format!("({})", direction)).style(ratatui::style::Color::Yellow),
+                //Span::from(" "),
                 Span::from(self.text)
             ]).into()
         } else {
@@ -80,9 +81,10 @@ struct App {
     recording_audio: bool,
     focus_state: FocusState,
 
-    audio_control_sink: watch::Sender<AudioRecordRequest>,
+    transcription: TranscriptionControl,
     prediction_request_sink: watch::Sender<StageActions>,
-    tts_request_sink: mpsc::Sender<String>,
+    audio: AudioInputControl,
+    tts: TtsControl,
     sys_message_sink: mpsc::Sender<String>
 }
 
@@ -93,7 +95,7 @@ enum FocusState {
 }
 
 impl App {
-    fn new(prediction_request_sink: watch::Sender<StageActions>, audio_control_sink: watch::Sender<AudioRecordRequest>, tts_request_sink: mpsc::Sender<String>, sys_message_sink: mpsc::Sender<String>, initial_direction: StageDirection) -> Self {
+    fn new(prediction_request_sink: watch::Sender<StageActions>, audio: AudioInputControl, transcription: TranscriptionControl, tts: TtsControl, sys_message_sink: mpsc::Sender<String>, initial_direction: StageDirection) -> Self {
         Self {
             scene: Default::default(),
             direction: initial_direction,
@@ -106,25 +108,67 @@ impl App {
             prediction_request_sink,
             is_requesting: false,
             audio_level: -60.,
+            audio,
             recording_audio: false,
-            audio_control_sink,
+            transcription,
             focus_state: FocusState::UserInput,
-            tts_request_sink,
+            tts,
             sys_message_sink
         }
     }
 
+    fn format_line<'a>(entry: &ConversationEntry, max_width: usize) -> Text<'a> {
+        let prefix = match entry {
+            ConversationEntry::Eva(_) => "Eva: ",
+            ConversationEntry::User(_) => "Argee: ",
+            ConversationEntry::ShipComputer(_) => "Ship Computer: ",
+            _ => "",
+        };
+
+        let style = match entry {
+            ConversationEntry::Eva(_) => Style::new().fg(style::Color::Cyan),
+            ConversationEntry::User(_) => Style::new().fg(style::Color::Magenta),
+            ConversationEntry::ShipComputer(_) => Style::new().fg(style::Color::Green),
+            ConversationEntry::StageDirection(_) => Style::new().fg(style::Color::Yellow),
+            ConversationEntry::SystemMessage(_) => Style::new().fg(style::Color::DarkGray),
+        };
+
+        let text_style = match entry {
+            ConversationEntry::StageDirection(_) => Style::new().fg(style::Color::Yellow),
+            ConversationEntry::SystemMessage(_) => Style::new().fg(style::Color::DarkGray),
+            _ => Style::new()
+        };
+
+        let text = match entry {
+            ConversationEntry::Eva(text) => text,
+            ConversationEntry::ShipComputer(text) => text,
+            ConversationEntry::StageDirection(text) => text,
+            ConversationEntry::SystemMessage(text) => text,
+            ConversationEntry::User(text) => text
+        };
+
+        let avail_width = max_width - prefix.len();
+        let indent = " ".repeat(prefix.len());
+        let wrap_options = textwrap::Options::new(avail_width).initial_indent(prefix).subsequent_indent(&indent);
+        let wrapped: Vec<Line> = textwrap::wrap(text, wrap_options)
+            .iter()
+            .enumerate()
+            .map(|(idx, s)| {
+                if idx == 0 {
+                    Line::from_iter([Span::from(prefix).style(style), Span::from(s[prefix.len()..].to_string()).style(text_style)])
+                } else {
+                    Line::from(s.to_string()).style(text_style)
+                }
+            }).collect();
+
+        Text::from_iter(wrapped)
+    }
+
     fn draw_conversation(&mut self, frame: &mut Frame, area: Rect) {
-        let items: Vec<Line> = self.scene.conversation().iter().rev().map(|entry| {
-            match entry {
-                ConversationEntry::User(text) => Line::from_iter([Span::from("Argee: ").style(ratatui::style::Color::Magenta), Span::from(text)]),
-                ConversationEntry::Eva(text) => Line::from_iter([Span::from("Eva: ").style(ratatui::style::Color::Cyan), Span::from(text)]),
-                ConversationEntry::ShipComputer(text) => Line::from_iter([Span::from("Ship Computer: ").style(ratatui::style::Color::Green), Span::from(text)]),
-                ConversationEntry::StageDirection(text) => Line::from_iter([text]).style(ratatui::style::Color::Yellow),
-                ConversationEntry::SystemMessage(text) => Line::from_iter([text]).style(ratatui::style::Color::DarkGray)
-            }
+        let width = area.width.into();
+        let items: Vec<Text> = self.scene.conversation().iter().rev().map(|entry| {
+            Self::format_line(entry, width)
         }).collect();
-        // FIXME: We need to somehow make long list items wrap. https://github.com/ratatui/ratatui/issues/128#issuecomment-1613918499
         // TODO: Would be nice to be able to scroll a longer conversation with the scroll wheel, or with page up/down
         frame.render_stateful_widget(
             List::new(items)
@@ -138,15 +182,44 @@ impl App {
     }
 
     fn draw_options(&mut self, frame: &mut Frame, area: Rect) {
-        frame.render_stateful_widget(
-            List::new(self.scene.reply_options().clone())
-                .block(Block::bordered().border_style(style::Color::LightGreen).title("Reply Options (Press 'Ctrl+R' to regenerate, Ctrl+Enter to use)"))
-                .style(ratatui::style::Color::White)
-                .highlight_symbol("> ")
-                .highlight_style(style::Style::new().bold().fg(style::Color::Cyan)),
-            area,
-            &mut self.reply_state
-        );
+        let borders = Block::bordered().border_style(style::Color::LightGreen).title("Reply Options (Press 'Ctrl+R' to regenerate, Ctrl+Enter to use)");
+        if self.scene.reply_options().len() == 0 && self.is_requesting {
+            let list = SkeletonText::new(std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH).unwrap().as_millis()  as u64)
+                .braille(true)
+                .line_widths(&[0.25, 0.5, 0.4, 0.6])
+                .mode(AnimationMode::Noise)
+                .block(borders);
+            frame.render_widget(list, area);
+        } else {
+            let wrap_options = textwrap::Options::new(area.width as usize).subsequent_indent("...");
+            let options: Vec<Text> = self.scene.reply_options().iter().map(|option| {
+                let mut contents: Vec<Line> = vec![];
+
+                if let Some(direction) = &option.stage_direction {
+                    let padded = format!("({})", direction);
+                    let mut wrapped_direction: Vec<Line> = textwrap::wrap(&padded, wrap_options.clone())
+                            .iter()
+                            .map(|x| { Line::from(x.to_string()).fg(style::Color::Yellow)}).collect();
+                    contents.append(&mut wrapped_direction);
+                }
+
+                let mut text: Vec<Line> = textwrap::wrap(&option.text, wrap_options.clone())
+                            .iter()
+                            .map(|x| { Line::from(x.to_string())}).collect();
+                contents.append(&mut text);
+                Text::from_iter(contents)
+            }).collect();
+            frame.render_stateful_widget(
+                List::new(options)
+                    .block(borders)
+                    .style(ratatui::style::Color::White)
+                    .highlight_symbol("> ".fg(Color::Cyan))
+                    .highlight_style(style::Style::new().bold().bg(style::Color::DarkGray))
+                    .repeat_highlight_symbol(true),
+                area,
+                &mut self.reply_state
+            );
+        }
     }
 
     fn draw_user_input(&mut self, frame: &mut Frame, area: Rect) {
@@ -170,24 +243,40 @@ impl App {
 
     fn draw_status(&self, frame: &mut Frame, area: Rect) {
         let minutes_remaining = self.direction.time_remaining.num_seconds() / 60;
-        let time_style = if minutes_remaining == 0 {
-            Style::new().fg(ratatui::style::Color::Red).bold().rapid_blink()
-        } else if minutes_remaining <= 5 {
-            Style::new().fg(ratatui::style::Color::Red).bold().slow_blink()
-        } else if minutes_remaining <= 10 {
+
+        let negative = self.direction.time_remaining.abs() != self.direction.time_remaining;
+
+
+        let time_style = if minutes_remaining <= 0 || negative {
+            Style::new().fg(ratatui::style::Color::LightRed).bold()
+        } else if minutes_remaining < 5 {
+            Style::new().fg(ratatui::style::Color::LightRed).bold()
+        } else if minutes_remaining < 10 {
             ratatui::style::Color::Red.into()
-        } else if minutes_remaining <= 25 {
+        } else if minutes_remaining < 25 {
             ratatui::style::Color::Yellow.into()
-        } else if minutes_remaining <= 60 {
+        } else if minutes_remaining < 60 {
             ratatui::style::Color::Green.into()
         } else {
             ratatui::style::Color::Blue.into()
         };
+
+        let formatted_time = if negative {
+            format!("-{:0>2}:{:0>2}:{:0>2}", self.direction.time_remaining.num_hours().abs(), self.direction.time_remaining.num_minutes().abs()% 60, self.direction.time_remaining.num_seconds().abs() % 60)
+        } else {
+            format!("{:0>2}:{:0>2}:{:0>2}", self.direction.time_remaining.num_hours(), self.direction.time_remaining.num_minutes() % 60, self.direction.time_remaining.num_seconds() % 60)
+        };
+
         let status_line = Line::from_iter([
             Span::from(format!("Episode {}", self.direction.episode_number)).style(ratatui::style::Color::LightBlue),
             Span::from(" | ").style(ratatui::style::Color::DarkGray),
-            // FIXME: Looks weird with negative numbers, and it doesn't actually blink in the vscode terminal.
-            Span::from(format!("Time Remaining: {:0>2}:{:0>2}:{:0>2}", self.direction.time_remaining.num_hours(), self.direction.time_remaining.num_minutes() % 60, self.direction.time_remaining.num_seconds() % 60)).style(time_style)
+            Span::from(format!("{} tracks", self.direction.current_playlist.len())).style(ratatui::style::Color::LightBlue),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("Time Remaining: {}", formatted_time)).style(time_style),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("{} artifacts recorded", self.scene.scenery().artifacts.len())).style(ratatui::style::Color::LightBlue),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("{} tokens sacrificed", self.scene.tokens_consumed)).style(ratatui::style::Color::LightCyan),
         ]);
         frame.render_widget(status_line, area);
     }
@@ -240,13 +329,32 @@ impl App {
 
         let status_layout = Layout::default()
             .direction(Direction::Horizontal)
-            .constraints([Constraint::Max(3), Constraint::Fill(2), Constraint::Fill(1)])
+            .constraints([Constraint::Max(3), Constraint::Fill(2), Constraint::Max(13), Constraint::Min(50)])
             .split(layout[3]);
 
         self.draw_user_input(frame, layout[2]);
         self.draw_io_throbber(frame, status_layout[0]);
         self.draw_status(frame, status_layout[1]);
-        self.draw_volume(frame, status_layout[2]);
+        self.draw_recording_status(frame, status_layout[2]);
+        self.draw_volume(frame, status_layout[3]);
+    }
+
+    fn draw_recording_status(&self, frame: &mut Frame, area: Rect) {
+        frame.render_widget(Line::from_iter(
+            if self.recording_audio {
+                [
+                    Span::from(" "),
+                    Span::from(" Recording ").bg(style::Color::LightRed).fg(style::Color::White),
+                    Span::from(" ")
+                ]
+            } else {
+                [
+                    Span::from(" "),
+                    Span::from("  Stopped  ").bg(style::Color::DarkGray).fg(style::Color::White),
+                    Span::from(" ")
+                ]
+            }
+        ), area);
     }
 
     fn draw_volume(&self, frame: &mut Frame, area: Rect) {
@@ -279,10 +387,53 @@ impl App {
             self.next_actions.push(ConversationEntry::StageDirection(direction.clone()));
         }
         self.next_actions.push(ConversationEntry::Eva(selected.text.clone()));
-        self.speak(selected.text.clone()).await;
+        self.tts.speak(selected.text.clone()).await;
         self.regenerate_responses();
     }
 
+    async fn on_command(&mut self, command: &str) {
+        let mut parts = command.splitn(2, " ");
+        let command = parts.next().unwrap();
+        let arg = parts.next().unwrap_or("");
+        match command {
+            // FIXME: Need some new kind of /bandcamp command to force loading of specific urls
+            "/episode" => {
+                if let Ok(episode_number) = arg.trim().parse::<u32>() {
+                    self.direction.episode_number = episode_number;
+                    self.sys_message_sink.send(format!("Updated episode number: {}", self.direction.episode_number)).await.unwrap();
+                    self.reload_mixxx_playlist();
+                } else {
+                    self.sys_message_sink.send("Invalid episode number format. Use /episode [number]".into()).await.unwrap();
+                    return;
+                }
+            },
+            "/timer" => {
+                if let Ok(minutes) = arg.trim().parse::<i64>() {
+                    self.end_time = Utc::now() + Duration::minutes(minutes);
+                    self.sys_message_sink.send(format!("Set timer for {} minutes.", minutes)).await.unwrap();
+                } else {
+                    self.sys_message_sink.send("Invalid timer format. Use /timer [minutes]".into()).await.unwrap();
+                }
+            },
+            "/narrative" => {
+                self.direction.narrative = arg.to_string();
+                self.sys_message_sink.send(format!("Updated stage direction: {}", self.direction.narrative)).await.unwrap();
+                self.regenerate_responses();
+            },
+            "/event" => {
+                self.next_actions.push(ConversationEntry::StageDirection(arg.to_string()));
+                self.regenerate_responses();
+            },
+            "/computer" => {
+                self.next_actions.push(ConversationEntry::ShipComputer(arg.to_string()));
+                self.regenerate_responses();
+            },
+            _ => {
+                self.sys_message_sink.send("Unknown command. Available commands: /episode [number], /narrative [text], /event [text], /computer [text], /timer [minutes]".into()).await.unwrap();
+            }
+        }
+    }
+
     async fn on_event(&mut self, evt: event::Event) {
         if let Some(key) = evt.as_key_press_event() {
             match self.focus_state {
@@ -291,6 +442,7 @@ impl App {
                         KeyCode::Tab => {
                             self.focus_state = FocusState::UserInput;
                             self.conversation_state.select(None);
+                            self.reply_state.select_first();
                         },
                         KeyCode::PageUp => self.conversation_state.scroll_down_by(5),
                         KeyCode::PageDown => self.conversation_state.scroll_up_by(5),
@@ -301,9 +453,10 @@ impl App {
                         KeyCode::Enter => {
                             let row_num = self.conversation_state.selected().unwrap();
                             if let ConversationEntry::Eva(text) = &self.scene.conversation()[self.scene.conversation().len() - 1 - row_num] {
-                                self.speak(text.clone()).await;
+                                self.tts.speak(text.clone()).await;
                                 self.focus_state = FocusState::UserInput;
                                 self.conversation_state.select(None);
+                                self.reply_state.select_first();
                             }
                         },
                         _ => ()
@@ -314,16 +467,17 @@ impl App {
                         KeyCode::Tab => {
                             self.focus_state = FocusState::Conversation;
                             self.conversation_state.select_first();
+                            self.reply_state.select(None);
                         },
                         KeyCode::Char('r') if key.modifiers.contains(KeyModifiers::CONTROL) => self.regenerate_responses(),
                         KeyCode::Char('x') if key.modifiers.contains(KeyModifiers::CONTROL) => {
                             if self.recording_audio {
                                 self.recording_audio = false;
-                                self.audio_control_sink.send_replace(AudioRecordRequest::Finish);
+                                self.transcription.stop();
                                 self.is_requesting = true;
                             } else {
                                 self.recording_audio = true;
-                                self.audio_control_sink.send_replace(AudioRecordRequest::Start);
+                                self.transcription.start();
                             }
                         },
                         KeyCode::Down => self.reply_state.select_next(),
@@ -337,70 +491,7 @@ impl App {
                                 self.insert_selected_prompt().await;
                             } else {
                                 if next_msg.starts_with("/") {
-                                    let mut parts = next_msg.splitn(2, " ");
-                                    let command = parts.next().unwrap();
-                                    let arg = parts.next().unwrap_or("");
-                                    match command {
-                                        "/bandcamp" => {
-                                            self.add_bandcamp_artifact(arg).await;
-                                            self.sys_message_sink.send(format!("Added Bandcamp artifact from {}", arg)).await.unwrap();
-                                            self.next_actions.push(ConversationEntry::ShipComputer(format!("Incoming transmission from {}", arg)));
-                                            self.regenerate_responses();
-                                        },
-                                        "/episode" => {
-                                            if let Ok(episode_number) = arg.trim().parse::<u32>() {
-                                                self.direction.episode_number = episode_number;
-                                                self.sys_message_sink.send(format!("Updated episode number: {}", self.direction.episode_number)).await.unwrap();
-                                                self.reload_mixxx_playlist();
-                                            } else {
-                                                self.sys_message_sink.send("Invalid episode number format. Use /episode [number]".into()).await.unwrap();
-                                                return;
-                                            }
-                                        },
-                                        "/timer" => {
-                                            if let Ok(minutes) = arg.trim().parse::<i64>() {
-                                                self.end_time = Utc::now() + Duration::minutes(minutes);
-                                                self.sys_message_sink.send(format!("Set timer for {} minutes.", minutes)).await.unwrap();
-                                            } else {
-                                                self.sys_message_sink.send("Invalid timer format. Use /timer [minutes]".into()).await.unwrap();
-                                            }
-                                        }
-                                        "/clear" => {
-                                            match arg.trim() {
-                                                "playlist" => {
-                                                    self.direction.current_playlist.clear();
-                                                    self.sys_message_sink.send("Cleared current playlist.".into()).await.unwrap();
-                                                },
-                                                "artifacts" => {
-                                                    self.direction.artifacts.clear();
-                                                    self.sys_message_sink.send("Cleared artifacts.".into()).await.unwrap();
-                                                },
-                                                "all" => {
-                                                    self.scene = Scene::default();
-                                                    self.sys_message_sink.send("Cleared all data.".into()).await.unwrap();
-                                                },
-                                                _ => {
-                                                    self.sys_message_sink.send("Unknown clear command. Use /clear [playlist|artifacts|all]".into()).await.unwrap();
-                                                }
-                                            }
-                                            return;
-                                        },
-                                        "/narrative" => {
-                                            self.direction.narrative = arg.to_string();
-                                            self.sys_message_sink.send(format!("Updated stage direction: {}", self.direction.narrative)).await.unwrap();
-                                        },
-                                        "/event" => {
-                                            self.next_actions.push(ConversationEntry::StageDirection(arg.to_string()));
-                                            self.regenerate_responses();
-                                        },
-                                        "/computer" => {
-                                            self.next_actions.push(ConversationEntry::ShipComputer(arg.to_string()));
-                                            self.regenerate_responses();
-                                        },
-                                        _ => {
-                                            self.sys_message_sink.send("Unknown command. Available commands: /bandcamp [url], /episode [number], /narrative [text], /reset".into()).await.unwrap();
-                                        }
-                                    }
+                                    self.on_command(&next_msg).await;
                                 } else {
                                     self.next_actions.push(ConversationEntry::User(next_msg));
                                     self.regenerate_responses();
@@ -414,19 +505,6 @@ impl App {
         }
     }
 
-    async fn add_bandcamp_artifact(&mut self, url: &str) {
-        // FIXME: This can crash if the page doesn't load properly, or if the structure of the Bandcamp page changes. We should add some error handling here.
-        let body  = reqwest::get(url).await.unwrap().text().await.unwrap();
-        let fragment = Html::parse_document(&body);
-        let selector = Selector::parse("script[type=\"application/ld+json\"]").unwrap();
-        let json_ld = fragment.select(&selector).next().unwrap().inner_html();
-        self.direction.artifacts.push(json_ld.trim().to_string());
-    }
-
-    async fn speak(&mut self, text: String) {
-        self.tts_request_sink.send(text).await.unwrap();
-    }
-
     fn regenerate_responses(&mut self) {
         let actions = StageActions {
             direction: self.direction.clone(),
@@ -439,29 +517,11 @@ impl App {
     }
 
     fn reload_mixxx_playlist(&mut self) {
-        // TODO: Should have some status message which states how many tracks are in the playlist
-        self.direction.current_playlist.clear();
-        let connection = sqlite::Connection::open_thread_safe_with_flags("mixxxdb.sqlite", OpenFlags::new().with_read_only()).unwrap();
-        let query = "SELECT id FROM Playlists WHERE name = ? ORDER BY id DESC LIMIT 1";
-        let mut statement = connection.prepare(query).unwrap();
-        statement.bind((1, format!("BFF.fm - Episode {}", self.direction.episode_number).as_str())).unwrap();
-        statement.next().unwrap();
-        let latest_id = statement.read::<i64, _>("id").unwrap();
-
-        let query = "SELECT title, artist, album, comment, url, bpm FROM library LEFT JOIN PlaylistTracks ON PlaylistTracks.track_id = library.id WHERE PlaylistTracks.playlist_id = ? ORDER BY position";
-        for track in connection.prepare(query).unwrap().into_iter().bind((1, latest_id)).unwrap().map(|row| row.unwrap()) {
-            let title = track.try_read::<&str, _>("title").unwrap_or("Untitled Track");
-            let artist = track.try_read::<&str, _>("artist").unwrap_or("Unknown Artist");
-            let album = track.try_read::<&str, _>("album").unwrap_or("Unknown Album");
-            let bpm = track.try_read::<f64, _>("bpm").unwrap_or(0.);
-            self.direction.current_playlist.push(PlaylistEntry {
-                artist: artist.into(),
-                album: album.into(),
-                title: title.into(),
-                bpm
-            });
+        if let Err(err) = self.direction.reload_mixxx_playlist() {
+            self.next_actions.push(ConversationEntry::SystemMessage(format!("Error while loading mixxx playlist: {:?}", err)));
+        } else {
+            self.next_actions.push(ConversationEntry::SystemMessage(format!("Mixxx playlist reloaded. {} tracks found.", self.direction.current_playlist.len()).into()));
         }
-        self.next_actions.push(ConversationEntry::SystemMessage("Mixxx playlist reloaded.".into()));
     }
 }
 
@@ -469,7 +529,8 @@ impl App {
 #[derive(Serialize, Deserialize, Debug, Default)]
 pub struct SaveData {
     pub direction: StageDirection,
-    pub messages: Vec<ChatCompletionRequestMessage>
+    pub messages: Vec<ChatCompletionRequestMessage>,
+    pub scenery: Scenery
 }
 
 impl SaveData {
@@ -481,28 +542,47 @@ impl SaveData {
 
 #[tokio::main]
 async fn main() {
-    color_eyre::install().unwrap();
+    let (panic_hook, eyre_hook) = color_eyre::config::HookBuilder::default()
+        .display_env_section(true)
+        .display_location_section(true)
+        .into_hooks();
+    eyre_hook.install().unwrap();
 
-    if option_env!("OPENAI_API_KEY").is_none() {
+    std::panic::set_hook(Box::new(move |panic_info| {
+        let msg = format!("{}", panic_hook.panic_report(panic_info));
+        println!("Panic: {}", msg);
+    }));
+
+    dotenv::dotenv().ok();
+
+    if std::env::var("OPENAI_API_KEY").is_err() {
         eprintln!("Error: OPENAI_API_KEY environment variable not set. The application will not function without it.");
         return;
     }
 
-    let saved_session = if let Ok(save_data) = std::fs::read_to_string("save.json") {
-        serde_json::from_str(&save_data).unwrap_or_default()
-        //FIXME: Re-add load messages to sys log
-    } else {
-        SaveData::default()
-    };
-
     let mut terminal: Terminal<CrosstermBackend<std::io::Stdout>> = ratatui::init();
 
     let (sys_message_sink, sys_message_src) = tokio::sync::mpsc::channel(32);
-    let tts_request_sender = start_tts().await;
-    let (prediction_request_in, mut prediction_out) = prediction::start_prediction(sys_message_src, saved_session.messages).await;
-    let (mut audio_state_receiver, audio_control_in, mut transcription_out) = transcription::start_transcription(sys_message_sink.clone()).await;
 
-    let mut app = App::new(prediction_request_in, audio_control_in, tts_request_sender, sys_message_sink, saved_session.direction);
+    let saved_session = if let Ok(save_data) = std::fs::read_to_string("save.json") {
+        if let Ok(ret) = serde_json::from_str(&save_data) {
+            sys_message_sink.send("Loaded session from save.json".into()).await.unwrap();
+            ret
+        } else {
+            sys_message_sink.send("Could not load saved session!".into()).await.unwrap();
+            SaveData::default()
+        }
+    } else {
+        sys_message_sink.send("Creating new session in save.json".into()).await.unwrap();
+        SaveData::default()
+    };
+
+    let (audio_ctrl, mic_stream, tts_output) = start_audio_input(&sys_message_sink).await;
+    let tts_ctrl = start_tts(tts_output).await;
+    let (prediction_request_in, mut prediction_out) = prediction::start_prediction(sys_message_src, saved_session.messages, saved_session.scenery).await;
+    let transcription_ctrl = transcription::start_transcription(mic_stream).await;
+
+    let mut app = App::new(prediction_request_in, audio_ctrl, transcription_ctrl, tts_ctrl, sys_message_sink, saved_session.direction);
 
     let mut events = EventStream::new();
     let mut last_tick = Instant::now();
@@ -525,13 +605,13 @@ async fn main() {
                 app.reply_state.select_first();
                 app.is_requesting = false;
             },
-            _ = audio_state_receiver.changed() => {
-                app.audio_level = *audio_state_receiver.borrow();
+            next_volume = app.audio.next() => {
+                app.audio_level = next_volume
             },
-            maybe_transcription = transcription_out.recv() => {
-                app.next_actions.push(ConversationEntry::User(maybe_transcription.unwrap()));
+            transcription_result = app.transcription.next() => {
+                app.next_actions.push(ConversationEntry::User(transcription_result));
                 app.regenerate_responses();
-            }
+            },
             maybe_event = event => {
                 match maybe_event {
                     Some(Ok(event)) => {

src/prediction.rs

@@ -1,12 +1,14 @@
 use std::process::{Command, Stdio};
 
-use async_openai::{Client, config::OpenAIConfig, types::chat::{ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestToolMessage, ChatCompletionRequestToolMessageArgs, ChatCompletionTool, ChatCompletionTools, CreateChatCompletionRequestArgs, FinishReason, FunctionObjectArgs, ResponseFormat, ResponseFormatJsonSchema}};
+use async_openai::{Client, config::OpenAIConfig, types::chat::{ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestToolMessageArgs, ChatCompletionTool, ChatCompletionTools, CreateChatCompletionRequestArgs, FinishReason, FunctionObjectArgs, ResponseFormat, ResponseFormatJsonSchema}};
 use bandcamp::SearchResultItem;
 use chrono::{DateTime, Utc};
+use color_eyre::eyre::eyre;
 use schemars::{JsonSchema, schema_for};
 use serde::{Deserialize, Serialize};
+use serde_json::{Serializer, ser::CompactFormatter};
 
-use crate::{SaveData, scene::{ConversationEntry, Scene, StageActions, StageDirection}};
+use crate::{SaveData, scene::{Artifact, ConversationEntry, Scene, Scenery, StageActions, StageDirection}};
 
 
 const SYSTEM_PROMPT: &str = include_str!("system-prompt.txt");
@@ -28,12 +30,20 @@ struct Session {
     conversation: Vec<ConversationEntry>,
     header_message: ChatCompletionRequestMessage,
     messages: Vec<ChatCompletionRequestMessage>,
-    reply_options: GeneratedResponses
+    reply_options: GeneratedResponses,
+    scenery: Scenery,
+    tokens_consumed: usize
 }
 
-#[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
+#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
+enum StageEvent {
+    ShipComputer(String),
+    StageDirection(String)
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
 struct StageEventArgs {
-    text: String,
+    event: StageEvent
 }
 
 #[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
@@ -51,7 +61,7 @@ struct BandcampQueryArgs {
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
-enum BandcampResult {
+pub enum BandcampResult {
     Artist { name: String, bio: Option<String>, location: Option<String> },
     Album { title: String, about: Option<String>, credits: Option<String>, release_date: DateTime<Utc>, artist: String }
 }
@@ -74,8 +84,14 @@ impl Into<BandcampResult> for bandcamp::Album {
     }
 }
 
+#[derive(Default, Debug)]
+struct ToolResults {
+    result: Option<String>,
+    messages: Vec<ConversationEntry>
+}
+
 impl Session {
-    fn from_initial_messages(messages: Vec<ChatCompletionRequestMessage>) -> Self {
+    fn from_initial_messages(messages: Vec<ChatCompletionRequestMessage>, scenery: Scenery) -> Self {
         let mut conversation = vec![];
         for msg in &messages {
             if let Ok(conversation_msg) = msg.clone().try_into() {
@@ -88,7 +104,9 @@ impl Session {
             conversation,
             header_message: ChatCompletionRequestSystemMessageArgs::default().content(SYSTEM_PROMPT).build().unwrap().into(),
             messages,
-            reply_options: Default::default()
+            reply_options: Default::default(),
+            scenery,
+            tokens_consumed: 0
         }
     }
 
@@ -98,14 +116,106 @@ impl Session {
         }
     }
 
+    async fn tool_stage_event(&mut self, args: StageEventArgs) -> ToolResults {
+        let msg = match args.event {
+            StageEvent::ShipComputer(text) => ConversationEntry::ShipComputer(text),
+            StageEvent::StageDirection(text) => ConversationEntry::StageDirection(text)
+        };
+        ToolResults {
+            messages: vec![msg],
+            ..Default::default()
+        }
+    }
+
+    async fn tool_bandcamp_scan(&mut self, args: BandcampQueryArgs) -> ToolResults {
+        let mut messages = vec![];
+        messages.push(ConversationEntry::SystemMessage(format!("Fetching artifacts from Bandcamp with {:?}", args).into()));
+        let mut json_results = vec![];
+        if let Ok(results) = bandcamp::search(args.query.as_str()).await {
+            for result in results {
+                match result {
+                    SearchResultItem::Artist(data) => {
+                        let result: BandcampResult = bandcamp::fetch_artist(data.artist_id).await.unwrap().into();
+                        json_results.push(Artifact::Bandcamp(result));
+                    },
+                    SearchResultItem::Album(data) => {
+                        let result: BandcampResult = bandcamp::fetch_album(data.band_id, data.album_id).await.unwrap().into();
+                        json_results.push(Artifact::Bandcamp(result));
+                    }
+                    _ => ()
+                }
+            }
+        }
+        let artifact_count = json_results.len();
+        messages.push(ConversationEntry::ShipComputer(format!("Relay scan for '{}' complete. {} artifacts added to the archive.", args.query, artifact_count).into()));
+
+        self.scenery.artifacts.append(&mut json_results);
+
+        ToolResults {
+            result: Some(format!("{} artifacts were added to the archive.", artifact_count)),
+            messages
+        }
+    }
+
+    async fn tool_artifact_query(&mut self, args: BeatsQueryArgs) -> ToolResults {
+        let mut messages = vec![];
+        let mut beets_cmd = Command::new("beet");
+        beets_cmd.args(["export", "-f", "json", "-i", "title,label,year,genres,album,artist"]);
+        if let Some(artist) = args.artist {
+            beets_cmd.arg(format!("artist:{}", artist));
+        }
+        if let Some(genre) = args.genre {
+            beets_cmd.arg(format!("genre:{}", genre));
+        }
+        if let Some(album) = args.album {
+            beets_cmd.arg(format!("album:{}", album));
+        }
+        if let Some(title) = args.title {
+            beets_cmd.arg(format!("title:{}", title));
+        }
+        if let Some(year) = args.year {
+            beets_cmd.arg(format!("year:{}", year));
+        }
+        if let Ok(output) = beets_cmd.stdout(Stdio::piped()).spawn().unwrap().wait_with_output() {
+            messages.push(ConversationEntry::ShipComputer(format!("Executing archive query {:?}", beets_cmd)));
+            self.scenery.artifacts.push(Artifact::BeetsTrack(serde_json::from_str(str::from_utf8(&output.stdout).unwrap()).unwrap()));
+        } else {
+            messages.push(ConversationEntry::ShipComputer("Unable to execute query!".into()));
+        };
+
+        ToolResults {
+            result: None,
+            messages
+        }
+    }
+
+    fn generate_conversation(&self, direction: &StageDirection) -> Vec<ChatCompletionRequestMessage> {
+        let mut json_buf = vec![];
+        let mut ser = Serializer::with_formatter(&mut json_buf, CompactFormatter);
+        direction.serialize(&mut ser).unwrap();
+        let direction_message: ChatCompletionRequestMessage = ChatCompletionRequestSystemMessageArgs::default()
+            .content(String::from_utf8(json_buf).unwrap())
+            .build().unwrap().into();
+
+        let mut json_buf = vec![];
+        let mut ser = Serializer::with_formatter(&mut json_buf, CompactFormatter);
+        self.scenery.serialize(&mut ser).unwrap();
+        let scenery_message: ChatCompletionRequestMessage = ChatCompletionRequestSystemMessageArgs::default()
+            .content(String::from_utf8(json_buf).unwrap())
+            .build().unwrap().into();
+        let mut full_conversation = vec![
+            self.header_message.clone(),
+            direction_message,
+            scenery_message,
+        ];
+        full_conversation.append(&mut self.messages.clone());
+
+        full_conversation
+    }
+
     async fn regenerate_options(&mut self, direction: &StageDirection) {
         loop {
-            let direction_message: ChatCompletionRequestMessage = ChatCompletionRequestSystemMessageArgs::default().content(serde_json::to_string(&direction).unwrap()).build().unwrap().into();
-            let mut full_conversation = vec![
-                self.header_message.clone(),
-                direction_message
-            ];
-            full_conversation.append(&mut self.messages.clone());
+            let full_conversation = self.generate_conversation(direction);
 
             let tools = vec![
                 ChatCompletionTools::Function(ChatCompletionTool {
@@ -115,13 +225,6 @@ impl Session {
                         .parameters(schema_for!(StageEventArgs))
                         .build().unwrap()
                 }),
-                ChatCompletionTools::Function(ChatCompletionTool {
-                    function: FunctionObjectArgs::default()
-                        .name("log_ship_computer_message")
-                        .description("Inserts a message from the ship computer into the scene script")
-                        .parameters(schema_for!(StageEventArgs))
-                        .build().unwrap()
-                }),
                 ChatCompletionTools::Function(ChatCompletionTool {
                     function: FunctionObjectArgs::default()
                         .name("archive_query")
@@ -132,14 +235,14 @@ impl Session {
                 ChatCompletionTools::Function(ChatCompletionTool {
                     function: FunctionObjectArgs::default()
                         .name("bandcamp_artifact_scan")
-                        .description("Scans Bandcamp to find artifacts to use in the scene that match the given search parameters")
+                        .description("Scans Bandcamp to find artifacts to use in the scene that match the given search parameters. To find an artist, provide only the artist name. To find an album, provide the artist and the album.")
                         .parameters(schema_for!(BandcampQueryArgs))
                         .build().unwrap()
                 })
             ];
             let request = CreateChatCompletionRequestArgs::default()
-                .messages(full_conversation.clone())
-                .model("gpt-5.4")
+                .messages(full_conversation)
+                .model("gpt-5.4-mini")
                 .tools(tools)
                 .max_completion_tokens(1024u32)
                 .response_format(ResponseFormat::JsonSchema {
@@ -153,9 +256,13 @@ impl Session {
                 .build().unwrap();
 
             let response = self.client.chat().create(request).await.unwrap_or_else(|err| {
-                panic!("{} {:?}", err, full_conversation);
+                panic!("OpenAI Panic: {}", err);
             });
 
+            if let Some(usage) = response.usage {
+                self.tokens_consumed += usage.total_tokens as usize;
+            }
+
             if let Some(message) = response.choices.first() {
 
                 match message.finish_reason {
@@ -176,100 +283,36 @@ impl Session {
                         .build().unwrap().into();
                     self.messages.push(assistant_messages);
                     let mut results = vec![];
-                    let mut messages = vec![];
                     for call in calls {
                         match call {
                             ChatCompletionMessageToolCalls::Function(call) => {
-                                match call.function.name.as_str() {
-                                    "log_stage_event" => {
-                                        let args: StageEventArgs = serde_json::from_str(call.function.arguments.as_str()).unwrap();
-                                        results.push(ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessageArgs::default()
-                                            .tool_call_id(call.id.clone())
-                                            .build().unwrap()
-                                        ));
-                                        messages.push(ConversationEntry::StageDirection(args.text));
-                                    },
-                                    "log_ship_computer_message" => {
-                                        let args: StageEventArgs = serde_json::from_str(call.function.arguments.as_str()).unwrap();
-                                        results.push(ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessageArgs::default()
-                                            .tool_call_id(call.id.clone())
-                                            .build().unwrap()
-                                        ));
-                                        messages.push(ConversationEntry::ShipComputer(args.text));
-                                    },
-                                    "bandcamp_artifact_scan" => {
-                                        let args: BandcampQueryArgs = serde_json::from_str(call.function.arguments.as_str()).unwrap();
-                                        self.insert_conversation(ConversationEntry::SystemMessage(format!("Fetching artifacts from Bandcamp with {:?}", args).into()));
-                                        let mut json_results = vec![];
-                                        if let Ok(results) = bandcamp::search(args.query.as_str()).await {
-                                            for result in results {
-                                                match result {
-                                                    SearchResultItem::Artist(data) => {
-                                                        let result: BandcampResult = bandcamp::fetch_artist(data.artist_id).await.unwrap().into();
-                                                        json_results.push(result);
-                                                    },
-                                                    SearchResultItem::Album(data) => {
-                                                        let result: BandcampResult = bandcamp::fetch_album(data.band_id, data.album_id).await.unwrap().into();
-                                                        json_results.push(result);
-                                                    }
-                                                    _ => ()
-                                                }
-                                            }
-                                        }
-                                        results.push(ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessageArgs::default()
-                                            .tool_call_id(call.id.clone())
-                                            .content(serde_json::to_string(&json_results).unwrap())
-                                            .build().unwrap()
-                                        ));
-                                        messages.push(ConversationEntry::ShipComputer(format!("Artifact scan for '{}' complete. {} results.", args.query, json_results.len()).into()));
-                                    },
-                                    "archive_query" => {
-                                        let args: BeatsQueryArgs = serde_json::from_str(call.function.arguments.as_str()).unwrap();
-                                        let mut beets_cmd = Command::new("beet");
-                                        beets_cmd.arg("export").arg("-f").arg("json").arg("-i").arg("title,label,year,genres,album,artist");
-                                        if let Some(artist) = args.artist {
-                                            beets_cmd.arg(format!("artist:{}", artist));
-                                        }
-                                        if let Some(genre) = args.genre {
-                                            beets_cmd.arg(format!("genre:{}", genre));
-                                        }
-                                        if let Some(album) = args.album {
-                                            beets_cmd.arg(format!("album:{}", album));
-                                        }
-                                        if let Some(title) = args.title {
-                                            beets_cmd.arg(format!("title:{}", title));
-                                        }
-                                        if let Some(year) = args.year {
-                                            beets_cmd.arg(format!("year:{}", year));
-                                        }
-                                        if let Ok(output) = beets_cmd.stdout(Stdio::piped()).spawn().unwrap().wait_with_output() {
-                                            let minified = minify::json::minify(str::from_utf8(&output.stdout).unwrap());
-                                            results.push(ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessageArgs::default()
-                                                .tool_call_id(call.id.clone())
-                                                .content(minified)
-                                                .build().unwrap()
-                                            ));
-                                            messages.push(ConversationEntry::ShipComputer(format!("Executing archive query {:?}", beets_cmd)));
-                                        } else {
-                                            messages.push(ConversationEntry::ShipComputer("Unable to execute query!".into()));
-                                            results.push(ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessageArgs::default()
-                                                .tool_call_id(call.id.clone())
-                                                .content("")
-                                                .build().unwrap()
-                                            ));
-                                        }
-                                    }
-                                    _ => panic!("Unknown function was called")
-                                }
+                                let func_name = call.function.name.as_str();
+                                let args = call.function.arguments.as_str();
+                                let tool_result = match func_name {
+                                    "log_stage_event" => self.tool_stage_event(serde_json::from_str(args).unwrap()).await,
+                                    "bandcamp_artifact_scan" => self.tool_bandcamp_scan(serde_json::from_str(args).unwrap()).await,
+                                    "archive_query" => self.tool_artifact_query(serde_json::from_str(args).unwrap()).await,
+                                    _ => unreachable!()
+                                };
+                                results.push((&call.id, tool_result));
                             },
                             _ => panic!("Unknown tool was called")
                         }
                     }
-                    self.messages.append(&mut results);
-                    for msg in messages {
+
+                    let mut tool_messages = vec![];
+                    for (id, mut result) in results {
+                        let mut msg = ChatCompletionRequestToolMessageArgs::default();
+                        msg.tool_call_id(id);
+                        if let Some(output) = result.result {
+                            msg.content(output);
+                        }
+                        self.messages.push(ChatCompletionRequestMessage::Tool(msg.build().unwrap()));
+                        tool_messages.append(&mut result.messages);
+                    }
+                    for msg in tool_messages {
                         self.insert_conversation(msg);
                     }
-
                 }
                 if let Some(content) = message.message.content.as_ref() {
                     if let Ok(options) = serde_json::from_str(content.as_str()) {
@@ -286,7 +329,7 @@ impl Session {
     }
 
     fn as_scene(&self) -> Scene {
-        Scene::new(self.reply_options.clone(), self.conversation.clone())
+        Scene::new(self.reply_options.clone(), self.conversation.clone(), self.scenery.clone(), self.tokens_consumed)
     }
 
     fn insert_conversation(&mut self, entry: ConversationEntry) {
@@ -298,11 +341,11 @@ impl Session {
     }
 }
 
-pub async fn start_prediction(mut sys_message_src: tokio::sync::mpsc::Receiver<String>, initial_messages: Vec<ChatCompletionRequestMessage>) -> (tokio::sync::watch::Sender<StageActions>, tokio::sync::watch::Receiver<Scene>) {
+pub async fn start_prediction(mut sys_message_src: tokio::sync::mpsc::Receiver<String>, initial_messages: Vec<ChatCompletionRequestMessage>, scenery: Scenery) -> (tokio::sync::watch::Sender<StageActions>, tokio::sync::watch::Receiver<Scene>) {
     let (prediction_in, prediction_out) = tokio::sync::watch::channel(Scene::default());
     let (prediction_request_in, mut prediction_request_out) = tokio::sync::watch::channel(StageActions::default());
 
-    let mut session = Session::from_initial_messages(initial_messages);
+    let mut session = Session::from_initial_messages(initial_messages, scenery);
 
     // Send the initial scene to the UI, after we have loaded the session from the first messages.
     prediction_in.send(session.as_scene()).unwrap();
@@ -323,7 +366,8 @@ pub async fn start_prediction(mut sys_message_src: tokio::sync::mpsc::Receiver<S
 
                         let mut save_data = SaveData {
                             direction: next_cxt.direction,
-                            messages: session.messages.clone()
+                            messages: session.messages.clone(),
+                            scenery: session.scenery.clone()
                         };
 
                         save_data.save();

src/scene.rs

@@ -1,8 +1,9 @@
 use async_openai::types::chat::*;
 use chrono::Duration;
 use serde::{Deserialize, Serialize};
+use sqlite::OpenFlags;
 
-use crate::prediction::{GeneratedResponses, PossibleResponse};
+use crate::prediction::{BandcampResult, GeneratedResponses, PossibleResponse};
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub enum ConversationEntry {
@@ -20,7 +21,7 @@ impl TryInto<ChatCompletionRequestMessage> for ConversationEntry {
             ConversationEntry::Eva(text) => Ok(ChatCompletionRequestMessage::Assistant(ChatCompletionRequestAssistantMessage { content: Some(text.into()), ..Default::default()})),
             ConversationEntry::ShipComputer(text) => Ok(ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: text.into(), name: Some("ship-computer".into()), ..Default::default() })),
             ConversationEntry::StageDirection(text) => Ok(ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: text.into(), name: Some("stage-direction".into()), ..Default::default() })),
-            ConversationEntry::SystemMessage(_) => Err(())
+            _ => Err(())
         }
     }
     
@@ -52,19 +53,58 @@ pub struct StageDirection {
     pub episode_number: u32,
     pub time_remaining: Duration,
     pub narrative: String,
-    pub artifacts: Vec<String>,
     pub current_playlist: Vec<PlaylistEntry>
 }
 
-/*impl StageDirection {
-    pub fn insert_conversation(&mut self, entry: ConversationEntry) {
-        self.additions.push(entry);
-    }
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Artifact {
+    Bandcamp(BandcampResult),
+    BeetsTrack(serde_json::Value)
+}
 
-    pub fn take_actions(&mut self) -> StageActions {
-        StageActions { direction: self.clone(), additions: std::mem::take(&mut self.additions) }
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+pub struct Scenery {
+    pub artifacts: Vec<Artifact>
+}
+
+#[derive(Debug)]
+pub enum MixxxError {
+    Sql(sqlite::Error)
+}
+
+impl From<sqlite::Error> for MixxxError {
+    fn from(value: sqlite::Error) -> Self {
+        Self::Sql(value)
     }
-}*/
+}
+
+impl StageDirection {
+    pub fn reload_mixxx_playlist(&mut self) -> Result<(), MixxxError> {
+        self.current_playlist.clear();
+        let connection = sqlite::Connection::open_thread_safe_with_flags("mixxxdb.sqlite", OpenFlags::new().with_read_only())?;
+        let query = "SELECT id FROM Playlists WHERE name = ? ORDER BY id DESC LIMIT 1";
+        let mut statement = connection.prepare(query)?;
+        statement.bind((1, format!("BFF.fm - Episode {}", self.episode_number).as_str()))?;
+        statement.next()?;
+        let latest_id = statement.read::<i64, _>("id").unwrap();
+
+        let query = "SELECT title, artist, album, comment, url, bpm FROM library LEFT JOIN PlaylistTracks ON PlaylistTracks.track_id = library.id WHERE PlaylistTracks.playlist_id = ? ORDER BY position";
+        for track in connection.prepare(query).unwrap().into_iter().bind((1, latest_id)).unwrap().map(|row| row.unwrap()) {
+            let title = track.try_read::<&str, _>("title").unwrap_or("Untitled Track");
+            let artist = track.try_read::<&str, _>("artist").unwrap_or("Unknown Artist");
+            let album = track.try_read::<&str, _>("album").unwrap_or("Unknown Album");
+            let bpm = track.try_read::<f64, _>("bpm").unwrap_or(0.);
+            self.current_playlist.push(PlaylistEntry {
+                artist: artist.into(),
+                album: album.into(),
+                title: title.into(),
+                bpm
+            });
+        }
+
+        Ok(())
+    }
+}
 
 #[derive(Debug, Default, Clone)]
 pub struct StageActions {
@@ -84,15 +124,23 @@ pub struct PlaylistEntry {
 pub struct Scene {
     reply_options: GeneratedResponses,
     conversation: Vec<ConversationEntry>,
+    pub tokens_consumed: usize,
+    scenery: Scenery
 }
 
 impl Scene {
-    pub fn new(reply_options: GeneratedResponses, conversation: Vec<ConversationEntry>) -> Self {
+    pub fn new(reply_options: GeneratedResponses, conversation: Vec<ConversationEntry>, scenery: Scenery, tokens_consumed: usize) -> Self {
         Self {
             reply_options,
             conversation,
+            scenery,
+            tokens_consumed
         }
     }
+
+    pub fn scenery(&self) -> &Scenery {
+        &self.scenery
+    }
     
     pub fn conversation(&self) -> &Vec<ConversationEntry> {
         &self.conversation

src/system-prompt.txt

@@ -45,7 +45,7 @@ It will also report out ship conditions, such as incoming transmissions, status
 # Constraints
 In a subsequent system prompt, you will be given the currrent 'stage direction' of the show, which includes the current playtime, the number of the episode, and any particular extra information about this episode that you should be aware of.
 The stage direction is provided as structured JSON. There may be additional data fields for semantic context that should be incorporated into the roleplaying setting.
-A list of artifacts that will be encountered during the episode are provided as blobs of json+ld metadata.
+A list of artifacts that will be encountered during the episode are provided as blobs of json metadata.
 Additionally, the current playlist of the radio show can be found as an array of track data.
 Your response will be used verbatim to generate speach using a text-to-speech engine, meaning you should not include any tone indicators or other formatting.
 This also means that your responses must not refer to any "lore", or "show", these instructions, or anything else out of character.

src/transcription.rs

@@ -1,12 +1,30 @@
 use std::{io::Read, sync::{Arc, Mutex}};
 
 use async_openai::{Client, config::OpenAIConfig, types::{InputSource, audio::{AudioInput, CreateTranscriptionRequest}}};
-use jack::{AudioIn, ClientOptions};
-use oximedia_metering::vu_meter::VuMeter;
 use tempfile::SpooledData;
 use tokio::sync::{mpsc, watch};
 
-use crate::events::AudioRecordRequest;
+use crate::{audio::MicStream, events::AudioRecordRequest};
+
+#[derive(Debug)]
+pub struct TranscriptionControl {
+    transcription_result_src: mpsc::Receiver<String>,
+    record_state_sink: watch::Sender<AudioRecordRequest>,
+}
+
+impl TranscriptionControl {
+    pub fn start(&mut self) {
+        self.record_state_sink.send(AudioRecordRequest::Start).unwrap();
+    }
+
+    pub fn stop(&mut self) {
+        self.record_state_sink.send(AudioRecordRequest::Finish).unwrap();
+    }
+
+    pub async fn next(&mut self) -> String {
+        self.transcription_result_src.recv().await.unwrap()
+    }
+}
 
 struct RcFile<T>(Arc<Mutex<T>>);
 
@@ -26,25 +44,24 @@ impl<T: std::io::Seek> std::io::Seek for RcFile<T> {
     }
 }
 
-pub async fn start_transcription(messages: mpsc::Sender<String>) -> (watch::Receiver<f64>, watch::Sender<AudioRecordRequest>, mpsc::Receiver<String>) {
-    let (audio_sink, mut audio_src) = mpsc::channel(32);
-    let (audio_state_sender, audio_state_receiver) = watch::channel(0.);
+pub async fn start_transcription(mut mic_src: MicStream) -> TranscriptionControl {
     let (audio_control_in, mut audio_control_out) = watch::channel(AudioRecordRequest::Finish);
     let (transcription_in, transcription_out) = mpsc::channel(1);
 
-    let rate = start_audio_input(&messages, audio_sink).await;
+    let ret = TranscriptionControl {
+        record_state_sink: audio_control_in,
+        transcription_result_src: transcription_out
+    };
 
     tokio::spawn(async move {
         let spec = hound::WavSpec {
             channels: 1,
-            sample_rate: rate,
+            sample_rate: mic_src.sample_rate,
             bits_per_sample: 16,
             sample_format: hound::SampleFormat::Int
         };
 
-        let mut meter = VuMeter::new(rate.into(), 1, None);
-
-        let spool_size = 16 * (rate as usize) * 10;// 10 seconds of audio
+        let spool_size = 16 * (mic_src.sample_rate as usize) * 10;// 10 seconds of audio
         let mut writer = None;
         let mut outfile = None;
 
@@ -85,10 +102,7 @@ pub async fn start_transcription(messages: mpsc::Sender<String>) -> (watch::Rece
                         }
                     }
                 },
-                maybe_audio_packet = audio_src.recv() => {
-                    let buf = maybe_audio_packet.unwrap();
-
-                    meter.process_interleaved(buf.as_slice());
+                Some(buf) = mic_src.src.recv() => {
                     if let Some(w) = writer.as_mut() {
                         for sample in buf.iter().copied() {
                             let sample_i16 = (sample * 32768.0)
@@ -98,43 +112,10 @@ pub async fn start_transcription(messages: mpsc::Sender<String>) -> (watch::Rece
                         }
                         w.flush().unwrap();
                     }
-                    audio_state_sender.send_if_modified(|v| {
-                        let next_vu = meter.channel_vu(0).unwrap();
-                        if *v != next_vu {
-                            *v = next_vu;
-                            true
-                        } else {
-                            false
-                        }
-                    });
                 }
             };
         }
     });
 
-    (audio_state_receiver, audio_control_in, transcription_out)
-}
-
-async fn start_audio_input(messages: &mpsc::Sender<String>, audio_sink: mpsc::Sender<Vec<f32>>) -> u32 {
-    let (client, _status) = jack::Client::new("Eva-Cohost", ClientOptions::default() | ClientOptions::SESSION_ID).unwrap();
-    let port = client.register_port("microphone-in", AudioIn::default()).unwrap();
-    let rate = client.sample_rate();
-
-    if let Ok(_) = client.connect_ports_by_name("mixxx-mic-1:capture_MONO", port.name().unwrap().as_str()) {
-        messages.send("Connected to audio.".into()).await.unwrap();
-    } else {
-        messages.send("Failed to reconnect to audio.".into()).await.unwrap();
-    }
-
-    let handler = jack::contrib::ClosureProcessHandler::new(move |_client, scope| {
-        if port.connected_count().unwrap() > 0 {
-            let buf: Vec<_> = port.as_slice(scope).iter().copied().collect();
-            audio_sink.blocking_send(buf).unwrap();
-        }
-        jack::Control::Continue
-    });
-    
-    std::mem::forget(client.activate_async((), handler).unwrap());
-
-    rate
+    ret
 }

src/tts.rs

@@ -1,16 +1,50 @@
-use std::process::Command;
+use std::process::{Command, Stdio};
 
-pub async fn start_tts() -> tokio::sync::mpsc::Sender<String> {
+use crate::audio::TtsOutStream;
+
+#[derive(Debug)]
+pub struct TtsControl {
+    request_sink: tokio::sync::mpsc::Sender<String>
+}
+
+impl TtsControl {
+    pub async fn speak(&self, text: String) {
+        self.request_sink.send(text).await.unwrap();
+    }
+}
+
+pub async fn start_tts(audio_sink: TtsOutStream) -> TtsControl {
 
     let (tts_request_sender, mut tts_request_receiver) = tokio::sync::mpsc::channel(3);
 
     // Set up the TTS task
     tokio::spawn(async move {
         while let Some(text) = tts_request_receiver.recv().await {
-            // TODO: We should also have espeak pipe out to stdout, then we can apply some audio effects and write to our own jack port.
-            Command::new("espeak-ng").arg("-v").arg("en-us+f3").arg(text).spawn().unwrap().wait().unwrap();
+            let tts_output = Command::new("espeak-ng").args(["-v", "en-us+f3", "--stdout"]).arg(text).stdout(Stdio::piped()).spawn().unwrap().wait_with_output().unwrap().stdout;
+            let tts_fd = std::io::Cursor::new(tts_output);
+            let mut wav_reader = hound::WavReader::new(tts_fd).unwrap();
+
+            let mut bitrate_resample = resampler::ResamplerFir::new_from_hz(1, wav_reader.spec().sample_rate, audio_sink.sample_rate, Default::default(), Default::default());
+
+            let mut audio_out_buf = vec![];
+
+            for sample in wav_reader.samples() {
+                if let Ok(raw_sample) = sample {
+                    let sample16: i16 = raw_sample;
+                    let sample32: f32 = (sample16 as f32) / (i16::MAX as f32);
+                    let mut audio_slice = [0.; 32];
+                    let (_, write_count) = bitrate_resample.resample(&[sample32], &mut audio_slice).unwrap();
+                    audio_out_buf.extend_from_slice(&audio_slice[0..write_count]);
+                } else {
+                    break;
+                }
+            }
+
+            audio_sink.sink.send(audio_out_buf).await.unwrap();
         }
     });
 
-    tts_request_sender
+    TtsControl {
+        request_sink: tts_request_sender
+    }
 }