shows: episode 206 in the bag

.gitignore

@@ -1,2 +1,6 @@
 /target
 save.json
+.env
+audio.json
+mixxxdb.sqlite*
+out.log

Cargo.toml

@@ -5,22 +5,43 @@ edition = "2024"
 
 [dependencies]
 async-openai = { version = "0.40.2", features = ["completions", "full"] }
+bandcamp = "0.3.4"
+better-panic = "0.3.0"
 chrono = { version = "0.4.44", features = ["serde"] }
 color-eyre = "0.6.5"
 crossterm = { version = "0.29.0", features = ["event-stream"] }
+dotenv = "0.15.0"
+enumset = { version = "1.1.13", features = ["serde"] }
 futures = "0.3.32"
 futures-timer = "3.0.4"
+hound = "3.5.1"
 iref = { version = "4.0.0", features = ["url", "serde"] }
+itertools = "0.15.0"
+jack = "0.13.5"
 json-ld = { version = "0.21.4", features = ["reqwest", "serde"] }
+log = "0.4.32"
+minify = "1.3.0"
+musicbrainz_rs = { version = "0.13.0", features = ["async"] }
+oximedia-metering = "0.1.7"
+rand = "0.10.1"
 ratatui = "0.30.0"
+rc-writer = "1.1.10"
 rdf-types = "0.22.5"
 reqwest = "0.13.4"
+resampler = "0.5.1"
 schemars = "1.2.1"
 scraper = "0.27.0"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.150"
 sqlite = "0.37.0"
 static-iref = "3.0.0"
+static_cell = "2.1.1"
+symphonia = { version = "0.6.0", features = ["all-codecs"] }
+tempfile = "3.27.0"
+textwrap = "0.16.2"
 throbber-widgets-tui = "0.11.0"
 tokio = { version = "1.52.3", features = ["full"] }
+tokio-stream = "0.1.18"
 tui-input = "0.15.3"
+tui-skeleton = "0.3.0"
+uuid = { version = "1.23.3", features = ["serde", "v4"] }

build.rs

@@ -0,0 +1,4 @@
+fn main() {
+    println!("cargo::rerun-if-changed=src/system-prompt.txt");
+    println!("cargo::rerun-if-changed=src/computer-prompt.txt");
+}

src/TODO.md

@@ -0,0 +1,44 @@
+# UI
+[ ] log line colors
+
+# Features
+[ ] pull lyrics and info from genius.com
+[ ] patch mixxx to expose a rest api for data
+[ ] MIDI input / output
+[ ] Different prompts between "recording an episode", "preparing for an episode", or "exploring"
+[x] Mixxx playlists other than the regular episode
+[ ] Include mixxx ratings into metadata
+[ ] Realtime updates to the mixxx playlist
+[ ] somehow optimize the massive json blob of data we send?
+
+# Refactoring
+[x] Move tools into a sub-module
+[x] Debug log should show up into the UI in a separate list
+[ ] Use pipewire over jack
+[x] split out UI into several widgets
+
+# Realtime behaviors
+[ ] "Hey eva, take a note, this one is pretty crunchy." -> note list in sidebar?
+[ ] "Hey eva, lets wrap up, can you prep me a report?" -> generates a summary of the episode, stashes it in the sidebar, saves it for the next episode, maybe also stores in beets/mixxx?
+[ ] "Hit it, eva" "Spin up the buffers, eva" - ship computer starts playing sfx to transition out
+[ ] ship should give off some warning sfx or something when the episode time hits 1h, 20m, 5m, 0, etc.
+[X] asking eva to query the archive or a relay or fetch something should be happening in the background, with the computer feeding updates into the conversation. Eva can then react as it goes along.
+[ ] Twitch chat?
+
+# Episode 205
+[x] Ambient SFX
+[x] Musicbrainz artifacts
+[x] Preload musicbrainz data from playlist with a command
+
+# Episode 206
+[ ] Automatic quiet detection instead of push-to-talk
+[ ] Fact database, to remember things about tracks during the show
+[ ] Conversation options that exceed a certain predicted amount of TTS time (1-2 seconds) should get highlighted differently
+[ ] Archive metadata and context should go at the very end of a prompt, to utilize token caching
+[ ] Report cache status to the UI
+[ ] "Environmental Status" panel that shows token usage as a sparkgraph, archive size, context window size
+[ ] Ship computer is kinda bad when confronted with non-research tasks
+[ ] Eva doesn't talk to the computer as much anymore for research tasks
+[ ] Save the computer task list to save.json
+[ ] Eva keeps saying "if you want a lore angle:" stuff
+[ ] Should be a kind of rapid-fire eva dialog mode, where eva immediately replies to voice prompts

src/artifacts/archive.rs

@@ -0,0 +1,190 @@
+use std::{collections::HashMap, ops::{Deref, DerefMut}};
+use std::fmt::Debug;
+
+use futures::StreamExt;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::artifacts::{Artifact, Contents, Merge, SourceID, beets::BeetsDB, mixxx::MixxxDB, musicbrainz::MBQuery, tools::DataSource};
+
+pub struct ArtifactRef<'a> {
+    id: Uuid,
+    archive: &'a Archive
+}
+
+pub struct ArtifactRefMut<'a> {
+    id: Uuid,
+    archive: &'a mut Archive
+}
+
+impl<'a> ArtifactRefMut<'a> {
+    pub fn downgrade(self) -> ArtifactRef<'a> {
+        ArtifactRef { id: self.id, archive: self.archive }
+    }
+}
+
+impl<'a> Deref for ArtifactRef<'a> {
+    type Target = Artifact;
+    fn deref(&self) -> &Self::Target {
+        self.archive.contents.get(&self.id).unwrap()
+    }
+}
+
+impl<'a> Deref for ArtifactRefMut<'a> {
+    type Target = Artifact;
+    fn deref(&self) -> &Self::Target {
+        self.archive.contents.get(&self.id).unwrap()
+    }
+}
+
+impl<'a> DerefMut for ArtifactRefMut<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.archive.contents.get_mut(&self.id).unwrap()
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct Archive {
+    #[serde(flatten)]
+    contents: HashMap<Uuid, Artifact>
+}
+
+impl Archive {
+    pub fn len(&self) -> usize {
+        self.contents.len()
+    }
+
+    // track, album, artist
+    pub fn stats(&self) -> (usize, usize, usize) {
+        self.contents.values().map(|artifact| {
+            match artifact.contents() {
+                Contents::Track(_) => (1, 0, 0),
+                Contents::Album(_) => (0, 1, 0),
+                Contents::Artist(_) => (0, 0, 1),
+            }
+        }).reduce(|acc, e| {
+            (acc.0 + e.0, acc.1 + e.1, acc.2 + e.2)
+        }).unwrap_or_default()
+    }
+
+    pub fn get<'a>(&'a self, id: &Uuid) -> Option<ArtifactRef<'a>> {
+        if self.contents.contains_key(id) {
+            Some(ArtifactRef { id: *id, archive: self })
+        } else {
+            None
+        }
+    }
+
+    pub fn get_mut<'a>(&'a mut self, id: &Uuid) -> Option<ArtifactRefMut<'a>> {
+        if self.contents.contains_key(id) {
+            Some(ArtifactRefMut { id: *id, archive: self })
+        } else {
+            None
+        }
+    }
+
+    pub async fn data_sync<Src: DataSource>(&mut self, datasrc: &mut Src, source: SourceID) -> usize where Src::Error: Debug {
+        let mut count = 0;
+
+        let pending = self.contents.iter_mut().filter_map(|(_, artifact)| {
+            if !artifact.sources.contains(&source) {
+                Some(artifact)
+            } else {
+                None
+            }
+        });
+
+        let futures = futures::stream::FuturesUnordered::new();
+
+        for artifact in pending {
+            futures.push(datasrc.synchronize(artifact));
+        }
+
+        let results: Vec<_> = futures.collect().await;
+        for result in results {
+            match result {
+                Ok(new_pending) => {
+                    count += new_pending.len() + 1;
+                    for new in new_pending {
+                        self.insert(new);
+                    }
+                },
+                Err(err) => {
+                    log::error!("Failed to synchronize: {:?}", err);
+                }
+            }
+        }
+
+        count
+    }
+
+    pub async fn synchronize(&mut self) -> usize {
+        log::debug!("Synchronizing records");
+        let mut count = 0;
+
+        log::debug!("Synchronizing Mixxx");
+        count += self.data_sync(&mut MixxxDB, SourceID::Mixxx).await;
+        log::debug!("Synchronizing Beets");
+        count += self.data_sync(&mut BeetsDB, SourceID::Beets).await;
+        log::debug!("Synchronizing Musicbrainz");
+        count += self.data_sync(&mut MBQuery, SourceID::Musicbrainz).await;
+
+        log::debug!("Updated {} records", count);
+
+        count
+    }
+
+    pub fn insert<'a>(&'a mut self, artifact: Artifact) -> ArtifactRef<'a> {
+        // If we are inserting a new artifact with a complete MBID...
+        if let Some(mbid) = artifact.mbid {
+            let search_id = mbid;
+            // If an entry already exists keyed by this MBID, merge into it
+            if let Some(existing) = self.contents.get_mut(&search_id) {
+                existing.merge(artifact);
+                ArtifactRef { id: search_id, archive: self }
+            } else {
+                // Otherwise, attempt to find existing artifacts with the same contents (but no MBID)
+                let mut targets: Vec<(Uuid, Artifact)> = self.contents.extract_if(|_, v| { v.contents == artifact.contents }).collect();
+                if let Some((_target_id, mut target)) = targets.pop() {
+                    // Merge any other extracted targets into the primary one
+                    for (_, next) in targets {
+                        target.merge(next);
+                    }
+                    // Merge the incoming artifact into the merged target
+                    target.merge(artifact);
+                    // Insert merged target under the canonical MBID key
+                    self.contents.insert(search_id, target);
+                    ArtifactRef { id: search_id, archive: self }
+                } else {
+                    // No matching content found: insert under the MBID key
+                    self.contents.insert(search_id, artifact);
+                    ArtifactRef { id: search_id, archive: self }
+                }
+            }
+        } else {
+            // Otherwise, we attempt to merge it in. In the end, there will somehow still be a record with this mbid
+            let mut targets: Vec<(Uuid, Artifact)> = self.contents.extract_if(|_, v| { v.contents == artifact.contents }).collect();
+            if let Some((target_id, mut target)) = targets.pop() {
+                let next_id = if let Some(mbid) = artifact.mbid {
+                    // If the new artifact has an mbid, we start using that as the archive key
+                    mbid
+                } else {
+                    // Otherwise, why regenerate a new one?
+                    target_id
+                };
+
+                for (_, next) in targets {
+                    target.merge(next);
+                }
+                target.merge(artifact);
+                // Re-insert the merged target back into the archive under the chosen id
+                self.contents.insert(next_id, target);
+                ArtifactRef { id: next_id, archive: self }
+            } else {
+                let new_id = Uuid::new_v4();
+                self.contents.insert(new_id, artifact);
+                ArtifactRef { id: new_id, archive: self }
+            }
+        }
+    }
+}

src/artifacts/bandcamp.rs

@@ -0,0 +1,87 @@
+use bandcamp::SearchResultItem;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::artifacts::{Album, Artifact, ArtifactBuilder, Artist, Contents, SourceID, Track, tools::{DataSource, ToolDescription}};
+
+#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
+pub struct BandcampQueryArgs {
+    pub query: String
+}
+
+impl From<bandcamp::Artist> for Artifact {
+    fn from(val: bandcamp::Artist) -> Self {
+        ArtifactBuilder::new(SourceID::Bandcamp).contents(Artist { name: val.name, bio: val.bio, location: val.location }).build()
+    }
+}
+
+impl From<bandcamp::Album> for Artifact {
+    fn from(val: bandcamp::Album) -> Self {
+        ArtifactBuilder::new(SourceID::Bandcamp)
+            .contents(Album {
+                about: val.about,
+                title: val.title,
+                artist: val.band.name,
+                credits: val.credits,
+                release_date: Some(val.release_date)
+            }).build()
+    }
+}
+
+pub struct BandcampSource;
+
+impl DataSource for BandcampSource {
+    type Args = BandcampQueryArgs;
+    type Error = bandcamp::Error;
+    
+    async fn synchronize(&self, artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
+        if let Contents::Artist(artist) = &artifact.contents {
+            Ok(self.query(&BandcampQueryArgs { query: artist.name.clone() }).await.unwrap_or_default())
+        } else {
+            Ok(vec![])
+        }
+    }
+    
+    async fn query(&self, args: &Self::Args) -> Result<Vec<Artifact>, Self::Error> {
+        log::debug!("Fetching artifacts from Bandcamp with {:?}", args);
+        let mut json_results = vec![];
+        if let Ok(results) = bandcamp::search(args.query.as_str()).await {
+            for result in results {
+                log::debug!("Result: {:?}", result);
+                match result {
+                    SearchResultItem::Artist(data) => {
+                        // TODO: The artist and album detailed fetchers should also be separate args
+                        let result = bandcamp::fetch_artist(data.artist_id).await?.into();
+                        json_results.push(result);
+                    },
+                    SearchResultItem::Album(data) => {
+                        let result = bandcamp::fetch_album(data.band_id, data.album_id).await?.into();
+                        json_results.push(result);
+                    },
+                    SearchResultItem::Track(data) => {
+                        let result = ArtifactBuilder::new(SourceID::Bandcamp)
+                            .contents(Track {
+                                title: data.name,
+                                artist: Some(data.band_name),
+                                album: data.album_name,
+                                ..Default::default()
+                            }).build();
+                        json_results.push(result);
+                    }
+                    _ => ()
+                }
+            }
+        }
+        Ok(json_results)
+    }
+}
+
+impl ToolDescription for BandcampSource {
+    fn description(&self) -> &str {
+        "Scans Bandcamp to find artifacts to use in the scene that match the given search parameters. To find an artist, provide only the artist name. To find an album, provide the artist and the album."
+    }
+    
+    fn name(&self) -> &str {
+        "query_bandcamp"
+    }
+}

src/artifacts/beets.rs

@@ -0,0 +1,386 @@
+use std::{collections::HashSet, fmt::Display, process::Stdio};
+
+use chrono::{DateTime, NaiveDate, NaiveTime, Utc};
+use serde_json::Value;
+use tokio::process::Command;
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Deserializer, Serialize, de::Visitor};
+use uuid::Uuid;
+
+use crate::artifacts::{Artifact, ArtifactBuilder, Contents, Merge, SourceID, Track, tools::{DataSource, ToolDescription}};
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub struct DateQuery {
+    year: u32,
+    month_and_day: Option<(u32, u32)>,
+    hour: Option<u32>,
+    minute: Option<u32>
+}
+
+impl Display for DateQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.year)?;
+        if let Some((month, day)) = self.month_and_day {
+            write!(f, "-{}-{}", month, day)?;
+        }
+
+        match (self.hour, self.minute) {
+            (Some(hour), None) => write!(f, "T{}", hour),
+            (Some(hour), Some(minute)) => write!(f, "T{}:{}", hour, minute),
+            (_, _) => Ok(())
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum RelativeDateUnit {
+    Day,
+    Week,
+    Month,
+    Year
+}
+
+impl Display for RelativeDateUnit {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Day => f.write_str("d"),
+            Self::Week => f.write_str("w"),
+            Self::Month => f.write_str("m"),
+            Self::Year => f.write_str("y")
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum RelativeDateQuery {
+    Earlier(u32, RelativeDateUnit),
+    Later(u32, RelativeDateUnit)
+}
+
+impl Display for RelativeDateQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Earlier(q, unit) => write!(f, "-{}{}", q, unit),
+            Self::Later(q, unit) => write!(f, "+{}{}", q, unit)
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum DateRangeQuery {
+    Before(DateQuery),
+    After(DateQuery),
+    #[serde(untagged)]
+    Between(DateQuery, DateQuery),
+    #[serde(untagged)]
+    Exact(DateQuery)
+}
+
+impl Display for DateRangeQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Exact(q) => write!(f, "{}", q),
+            Self::Before(q) => write!(f, "..{}", q),
+            Self::After(q) => write!(f, "{}..", q),
+            Self::Between(q, z) => write!(f, "{}..{}", q, z),
+        }
+    }
+}
+
+/// Matches the numeric value of a field against another number or range of numbers
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum NumericRangeQuery {
+    /// Matches all numbers equal to or greater than the provided value
+    Above(u32),
+    /// Matches all numbers equal to or less than the provided value
+    Below(u32),
+    #[serde(untagged)]
+    /// Matches all numbers within the given range, inclusive
+    Between(u32, u32),
+    /// Matches exactly this number
+    #[serde(untagged)]
+    Exact(u32)
+}
+
+impl Display for NumericRangeQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Exact(v) => write!(f, "{}", v),
+            Self::Above(v) => write!(f, "{}..", v),
+            Self::Below(v) => write!(f, "..{}", v),
+            Self::Between(v, z) => write!(f, "{}..{}", v, z)
+        }
+    }
+}
+
+/// Match the string value of a field in various case-insensitive ways
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum BeetsStringFieldMatch {
+    /// Matches a field against a regular expression
+    RegularExpression(String),
+    /// Performs full-text comparison
+    ExactMatch(String),
+    /// WIll match against a partial text comparison; if the provided value is somewhere within the field, it will match
+    #[serde(untagged)]
+    Substring(String),
+}
+
+impl Display for BeetsStringFieldMatch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::RegularExpression(q) => write!(f, ":{}", q),
+            Self::Substring(q) => f.write_str(q),
+            Self::ExactMatch(q) => write!(f, "=~{}", q),
+        }
+    }
+}
+
+/// Matches the date value of a field
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+#[serde(untagged)]
+pub enum BeetsDateFieldMatch {
+    /// Represents a known date in time, eg, "24/06/26 at 16:17"
+    Absolute(DateRangeQuery),
+    /// Represents a relative date in time, eg "three weeks ago"
+    Relative(RelativeDateQuery)
+}
+
+impl Display for BeetsDateFieldMatch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Absolute(q) => write!(f, "{}", q),
+            Self::Relative(q) => write!(f, "{}", q),
+        }
+    }
+}
+
+/// A single field
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+#[serde(tag = "field", content = "value")]
+pub enum BeetsFieldQuery {
+    Artist(BeetsStringFieldMatch),
+    Album(BeetsStringFieldMatch),
+    Genre(BeetsStringFieldMatch),
+    Title(BeetsStringFieldMatch),
+    Label(BeetsStringFieldMatch),
+
+    ReleaseDate(BeetsDateFieldMatch),
+    /// The last time this track was played
+    LastPlayed(BeetsDateFieldMatch),
+    /// The date that this track was first imported
+    Added(BeetsDateFieldMatch),
+
+    /// The "rating" of a track on a 5 point scale, with higher points meaning higher energy
+    Rating(NumericRangeQuery),
+
+    /// The number of times the track has been played
+    Playcount(NumericRangeQuery),
+
+    #[serde(skip)]
+    MixxxPlaylist(String)
+}
+
+impl Display for BeetsFieldQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Artist(q) => write!(f, "artist:{}", q),
+            Self::Album(q) => write!(f, "album:{}", q),
+            Self::Genre(q) => write!(f, "genres:{}", q),
+            Self::Title(q) => write!(f, "title:{}", q),
+            Self::Label(q) => write!(f, "label:{}", q),
+            Self::ReleaseDate(q) => write!(f, "date:{}", q),
+            Self::LastPlayed(q) => write!(f, "mixxx_last_played:{}", q),
+            Self::Added(q) => write!(f, "added:{}", q),
+            Self::Rating(q) => write!(f, "mixxx_rating:{}", q),
+            Self::Playcount(q) => write!(f, "mixxx_times_played:{}", q),
+            Self::MixxxPlaylist(q) => write!(f, "mixxx_playlist:{}", q)
+        }
+    }
+}
+
+/// A list of search parameters to query against the beets database.
+#[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
+pub struct BeetsQueryArgs {
+    pub search_fields: Vec<BeetsFieldQuery>
+}
+
+/// A list of beets queries to run in parallel
+#[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
+pub struct BeatsQueryMultiArgs {
+    args: Vec<BeetsQueryArgs>
+}
+
+#[derive(Debug, Default, Deserialize)]
+struct BeetsTrack {
+    title: String,
+    album: String,
+    artist: String,
+    genres: Option<HashSet<String>>,
+    label: Option<String>,
+    date: Option<BeetsTimestamp>,
+    mixxx_last_played: Option<BeetsTimestamp>,
+    added: Option<BeetsTimestamp>,
+    mixxx_rating: Option<Value>,
+    mixxx_times_played: Option<Value>,
+
+    mb_trackid: Option<String>
+}
+
+#[derive(Debug)]
+struct BeetsTimestamp(NaiveDate);
+
+struct BeetsTimestampVisitor;
+
+impl<'de> Visitor<'de> for BeetsTimestampVisitor {
+    type Value = BeetsTimestamp;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("a beets date format string %Y-%m-%d")
+    }
+    
+    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        log::debug!("visit str");
+        Ok(BeetsTimestamp(NaiveDate::parse_from_str(v, "%Y-%m-%d").unwrap().into()))
+    }
+}
+
+impl<'de> Deserialize<'de> for BeetsTimestamp {
+    
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de> {
+        deserializer.deserialize_str(BeetsTimestampVisitor)
+    }
+}
+
+impl From<BeetsTimestamp> for DateTime<Utc> {
+    fn from(val: BeetsTimestamp) -> Self {
+        val.0.and_time(NaiveTime::default()).and_utc()
+    }
+}
+
+impl From<BeetsTrack> for Artifact {
+    fn from(val: BeetsTrack) -> Self {
+        let track_data = Track {
+            title: val.title,
+            album: Some(val.album),
+            artist: Some(val.artist),
+            genres: val.genres.unwrap_or_default(),
+            label: val.label,
+            release_date: val.date.map(BeetsTimestamp::into),
+            last_played: val.mixxx_last_played.map(BeetsTimestamp::into),
+            added: val.added.map(BeetsTimestamp::into),
+            rating: val.mixxx_rating.map(|x| x.as_u64().unwrap_or_default() as u32 ),
+            bpm: None,
+        };
+        let builder = ArtifactBuilder::new(SourceID::Beets)
+            .contents(track_data);
+        if let Ok(mbid) = Uuid::parse_str(&val.mb_trackid.unwrap_or_default()) {
+            builder.mbid(mbid).build()
+        } else {
+            builder.build()
+        }
+    }
+}
+
+#[derive(Debug)]
+#[allow(unused)]
+pub enum BeetsError {
+    Command(std::io::Error),
+    Json(serde_json::Error),
+    EmptyQuery
+}
+
+impl From<std::io::Error> for BeetsError {
+    fn from(value: std::io::Error) -> Self {
+        Self::Command(value)
+    }
+}
+
+impl From<serde_json::Error> for BeetsError {
+    fn from(value: serde_json::Error) -> Self {
+        Self::Json(value)
+    }
+}
+
+pub struct BeetsDB;
+
+impl BeetsDB {
+    async fn query_multi(&self, args: &BeatsQueryMultiArgs) -> Result<Vec<Artifact>, BeetsError> {
+        let mut ret = vec![];
+        for arg in &args.args {
+            for artifact in self.query_single(arg).await.unwrap_or_default() {
+                ret.push(artifact);
+            }
+        }
+        Ok(ret)
+    }
+
+    async fn query_single(&self, args: &BeetsQueryArgs) -> Result<Vec<Artifact>, BeetsError> {
+        let mut beets_cmd = Command::new("beet");
+        beets_cmd.args(["export", "-f", "json", "-i", "title,artist,album,genres,label,date,mixxx_last_played,added,mixxx_rating,mixxx_times_played,mb_trackid"]);
+
+        if args.search_fields.is_empty() {
+            log::warn!("Tried to execute an empty beets query");
+            return Err(BeetsError::EmptyQuery)
+        }
+
+        for field in &args.search_fields {
+            beets_cmd.arg(format!("{}", field));
+        }
+
+        log::debug!("Executing beets: {:?}", beets_cmd);
+        let output = beets_cmd.stdout(Stdio::piped()).stderr(Stdio::null()).spawn()?.wait_with_output().await?;
+        log::debug!("Output: {}", str::from_utf8(&output.stdout).unwrap());
+        let track = serde_json::from_str::<Vec<BeetsTrack>>(str::from_utf8(&output.stdout).unwrap()).unwrap();
+        Ok(track.into_iter().map(BeetsTrack::into).collect())
+    }
+}
+
+impl DataSource for BeetsDB {
+    type Args = BeetsQueryArgs;
+
+    type Error = BeetsError;
+
+    async fn synchronize(&self, artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
+        if let Contents::Track(ref mut target_track) = artifact.contents {
+            let args = BeetsQueryArgs {
+                search_fields: vec![
+                    BeetsFieldQuery::Title(BeetsStringFieldMatch::Substring(target_track.title.clone())),
+                    BeetsFieldQuery::Artist(BeetsStringFieldMatch::Substring(target_track.artist.clone().unwrap_or_default())),
+                    BeetsFieldQuery::Album(BeetsStringFieldMatch::Substring(target_track.album.clone().unwrap_or_default()))
+                ]
+            };
+
+            let results = self.query(&args).await?;
+
+            if let Some(first) = results.first() {
+                artifact.merge(first.clone());
+            } else {
+                log::debug!("Beets could not find {:?}", target_track);
+            }
+        }
+
+        Ok(vec![])
+    }
+    
+    fn query(&self, args: &Self::Args) -> impl Future<Output = Result<Vec<Artifact>, Self::Error>> {
+        self.query_single(args)
+    }
+
+    
+}
+
+impl ToolDescription for BeetsDB {
+    fn description(&self) -> &str {
+        "Queries the ship's musical artifact archives for tracks matching the given search parameters"
+    }
+    
+    fn name(&self) -> &str {
+        "query_beets"
+    }
+}

src/artifacts/mixxx.rs

@@ -0,0 +1,87 @@
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use sqlite::OpenFlags;
+
+use crate::artifacts::{Album, Artifact, ArtifactBuilder, Artist, SourceID, Track, tools::{DataSource, ToolDescription}};
+
+#[derive(Debug)]
+#[allow(unused)]
+pub enum MixxxError {
+    Sql(sqlite::Error)
+}
+
+impl From<sqlite::Error> for MixxxError {
+    fn from(value: sqlite::Error) -> Self {
+        Self::Sql(value)
+    }
+}
+
+pub struct MixxxDB;
+
+#[derive(Serialize, Deserialize, Debug, Default, JsonSchema)]
+pub struct MixxxQuery {
+    pub playlist_name: String
+}
+
+impl DataSource for MixxxDB {
+    type Args = MixxxQuery;
+    type Error = MixxxError;
+
+    async fn synchronize(&self, _artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
+        Ok(vec![])
+    }
+
+    async fn query(&self, args: &Self::Args) -> Result<Vec<Artifact>, Self::Error> {
+        let mut ret = vec![];
+        let playlist_name = args.playlist_name.as_str();
+        log::info!("Loading Mixxx playlist {}", playlist_name);
+        let connection = sqlite::Connection::open_thread_safe_with_flags("/home/tdfischer/.mixxx/mixxxdb.sqlite", OpenFlags::new().with_read_only())?;
+        let query = "SELECT id FROM Playlists WHERE name = ? ORDER BY id DESC LIMIT 1";
+        let mut statement = connection.prepare(query)?;
+        statement.bind((1, playlist_name))?;
+        statement.next()?;
+        let latest_id = statement.read::<i64, _>("id")?;
+
+        let query = "SELECT title, artist, album, comment, url, bpm FROM library LEFT JOIN PlaylistTracks ON PlaylistTracks.track_id = library.id WHERE PlaylistTracks.playlist_id = ? ORDER BY position";
+        for track in connection.prepare(query)?.into_iter().bind((1, latest_id))? {
+            let track = track?;
+            let title = track.try_read::<&str, _>("title").unwrap_or("Untitled Track");
+            let artist = track.try_read::<&str, _>("artist").unwrap_or("Unknown Artist");
+            let album = track.try_read::<&str, _>("album").unwrap_or("Unknown Album");
+            let bpm = track.try_read::<f64, _>("bpm").unwrap_or(0.);
+            ret.push(ArtifactBuilder::new(SourceID::Mixxx)
+                .contents(Track {
+                    artist: Some(artist.into()),
+                    album: Some(album.into()),
+                    title: title.into(),
+                    bpm: Some(bpm),
+                    ..Default::default()
+                }).build());
+
+            ret.push(ArtifactBuilder::new(SourceID::Mixxx)
+                .contents(Album {
+                    artist: artist.into(),
+                    title: album.into(),
+                    ..Default::default()
+                }).build());
+
+            ret.push(ArtifactBuilder::new(SourceID::Mixxx)
+                .contents(Artist {
+                    name: artist.into(),
+                    ..Default::default()
+                }).build());
+        }
+
+        Ok(ret)
+    }
+}
+
+impl ToolDescription for MixxxDB {
+    fn description(&self) -> &str {
+        "Loads artifacts from a given Mixxx playlist name"
+    }
+    
+    fn name(&self) -> &str {
+        "query_mixxx"
+    }
+}

src/artifacts/mod.rs

@@ -0,0 +1,267 @@
+use std::{collections::HashSet, fmt::Debug, };
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+
+pub mod bandcamp;
+pub mod mixxx;
+pub mod beets;
+pub mod musicbrainz;
+pub mod archive;
+pub mod tools;
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum SourceID {
+    Bandcamp,
+    Musicbrainz,
+    Mixxx,
+    Beets
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
+pub struct Artist {
+    pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bio: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub location: Option<String>,
+}
+
+impl PartialEq for Artist {
+    fn eq(&self, other: &Self) -> bool {
+        if self.name != other.name {
+            return false;
+        }
+
+        true
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
+pub struct Album {
+    pub title: String,
+    pub artist: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub about: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub credits: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub release_date: Option<DateTime<Utc>>
+}
+
+impl PartialEq for Album {
+    fn eq(&self, other: &Self) -> bool {
+        if self.title != other.title || self.artist != other.artist {
+            return false;
+        }
+
+        true
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
+pub struct Track {
+    pub title: String,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub album: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub artist: Option<String>,
+
+    #[serde(skip_serializing_if = "HashSet::is_empty")]
+    #[serde(default)]
+    pub genres: HashSet<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub label: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub release_date: Option<DateTime<Utc>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_played: Option<DateTime<Utc>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub added: Option<DateTime<Utc>>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rating: Option<u32>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bpm: Option<f64>
+}
+
+impl PartialEq for Track {
+    fn eq(&self, other: &Self) -> bool {
+        if self.title != other.title {
+            return false;
+        }
+
+        if other.artist.is_some() && self.artist.is_some() && self.artist != other.artist {
+            return false;
+        }
+
+        if other.album.is_some() && self.album.is_some() && self.album != other.album {
+            return false;
+        }
+
+        true
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[serde(tag = "type")]
+pub enum Contents {
+    Artist(Artist),
+    Album(Album),
+    Track(Track)
+}
+
+impl From<Artist> for Contents {
+    fn from(value: Artist) -> Self {
+        Self::Artist(value)
+    }
+}
+
+impl From<Album> for Contents {
+    fn from(value: Album) -> Self {
+        Self::Album(value)
+    }
+}
+
+impl From<Track> for Contents {
+    fn from(value: Track) -> Self {
+        Self::Track(value)
+    }
+}
+
+impl<M: Merge + PartialEq> Merge for Vec<M> {
+    fn merge(&mut self, other: Self) {
+        for artifact in other {
+            if let Some(merge_target) = self.iter_mut().find(|a| { **a == artifact }) {
+                merge_target.merge(artifact);
+            } else {
+                self.push(artifact);
+            }
+        }
+    }
+}
+
+pub trait Merge {
+    fn merge(&mut self, other: Self);
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+pub struct Artifact {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    mbid: Option<Uuid>,
+    #[serde(flatten)]
+    contents: Contents,
+
+    sources: HashSet<SourceID>,
+}
+
+#[derive(Debug)]
+pub struct ArtifactBuilder {
+    contents: Option<Contents>,
+    mbid: Option<Uuid>,
+    source: SourceID,
+}
+
+impl ArtifactBuilder {
+    pub fn new(source: SourceID) -> Self {
+        Self {
+            contents: None,
+            mbid: None,
+            source,
+        }
+    }
+
+    pub fn contents<T: Into<Contents>>(mut self, contents: T) -> Self {
+        self.contents = Some(contents.into());
+        self
+    }
+
+    pub fn mbid<T: Into<Uuid>>(mut self, mbid: T) -> Self {
+        self.mbid = Some(mbid.into());
+        self
+    }
+
+    pub fn build(self) -> Artifact {
+        Artifact {
+            mbid: self.mbid,
+            contents: self.contents.unwrap(),
+            sources: HashSet::from_iter([self.source]),
+        }
+    }
+}
+
+impl Artifact {
+    pub fn contents(&self) -> &Contents {
+        &self.contents
+    }
+}
+
+impl Merge for Artifact {
+    fn merge(&mut self, other: Self) {
+        self.contents.merge(other.contents);
+        if self.mbid.is_none() {
+            self.mbid = other.mbid;
+        }
+        for src in other.sources {
+            self.sources.insert(src);
+        }
+    }
+}
+
+macro_rules! merge_fields {
+    ($this:expr, $that:expr, $field:tt) => {
+        if $this.$field.is_none() {
+            $this.$field = $that.$field;
+        }
+    };
+    ($this:tt, $that:tt, $($fields:tt),+) => {
+        $(
+            merge_fields!($this, $that, $fields);
+        )+
+    }
+}
+
+impl Merge for Contents {
+    fn merge(&mut self, other: Self) {
+        if *self != other {
+            return;
+        }
+
+        match (self, other) {
+            (Self::Track(this_track), Self::Track(that_track)) => {
+                this_track.merge(that_track);
+            },
+            (Self::Album(this_album), Self::Album(that_album)) => {
+                this_album.merge(that_album);
+            },
+            (Self::Artist(this_artist), Self::Artist(that_artist)) => {
+                this_artist.merge(that_artist);
+            },
+            _ => ()
+        }
+    }
+}
+
+impl Merge for Track {
+    fn merge(&mut self, other: Self) {
+        merge_fields!(self, other, album, label, release_date, artist, bpm, rating, added, last_played);
+    }
+}
+
+impl Merge for Artist {
+    fn merge(&mut self, other: Self) {
+        merge_fields!(self, other, bio, location);
+    }
+}
+
+impl Merge for Album {
+    fn merge(&mut self, other: Self) {
+        merge_fields!(self, other, about, credits, release_date);
+    }
+}

src/artifacts/musicbrainz.rs

@@ -0,0 +1,150 @@
+use musicbrainz_rs::entity::artist_credit::ArtistCredit;
+use musicbrainz_rs::entity::release::Release;
+use musicbrainz_rs::{ApiEndpointError, entity::recording::Recording};
+use musicbrainz_rs::prelude::*;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::artifacts::tools::{DataSource, ToolDescription};
+use crate::artifacts::{Album, Artifact, ArtifactBuilder, Artist, Contents, Merge, SourceID, Track};
+
+impl From<Recording> for Track {
+    fn from(value: Recording) -> Self {
+        let artist = value.artist_credit.unwrap_or_default().first().map(|x| x.name.clone() );
+        let album = value.releases.unwrap_or_default().first().map(|x| x.title.clone() );
+
+        Self {
+            title: value.title,
+            artist,
+            album,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<Release> for Album {
+    fn from(value: Release) -> Self {
+        let artist = value.artist_credit.unwrap_or_default().first().map(|x| x.name.clone() );
+
+        Self {
+            about: value.annotation,
+            title: value.title,
+            artist: artist.unwrap_or_default(),
+            ..Default::default()
+        }
+    }
+}
+
+impl From<ArtistCredit> for Artist {
+    fn from(value: ArtistCredit) -> Self {
+        Self {
+            bio: value.artist.annotation,
+            location: value.artist.country,
+            name: value.name,
+        }
+    }
+}
+
+pub struct MBQuery;
+
+impl MBQuery {
+    fn extract_recording_data(track: Recording) -> (Artifact, Vec<Artifact>) {
+        let mut ret = vec![];
+
+        let ret_track = ArtifactBuilder::new(SourceID::Musicbrainz)
+            .contents(Track::from(track.clone()))
+            .mbid(Uuid::parse_str(&track.id).unwrap())
+            .build();
+
+        for release in track.releases.unwrap_or_default() {
+            log::debug!("Found new release: {:?}", release);
+            ret.push(ArtifactBuilder::new(SourceID::Musicbrainz)
+                .mbid(Uuid::parse_str(&release.id).unwrap())
+                .contents(Album::from(release))
+                .build());
+        }
+        for artist in track.artist_credit.unwrap_or_default() {
+            ret.push(ArtifactBuilder::new(SourceID::Musicbrainz)
+                .mbid(Uuid::parse_str(&artist.artist.id).unwrap())
+                .contents(Artist::from(artist))
+                .build());
+        }
+
+        (ret_track, ret)
+    }
+}
+
+impl DataSource for MBQuery {
+    type Error = ApiEndpointError;
+    type Args = MusicbrainzQueryArgs;
+
+    async fn synchronize(&self, artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
+        let mut ret = vec![];
+        if artifact.mbid.is_none() {
+            return Ok(ret);
+        }
+        let artifact_id = artifact.mbid.unwrap();
+        log::debug!("Synchronizing {} with musicbrainz", artifact_id);
+        // FIXME: Need to also synchronize albums and artists
+        if let Contents::Track(_) = artifact.contents {
+            let mb_track = Recording::fetch()
+                .id(&artifact_id.to_string())
+                .with_releases().with_artists().with_annotations().execute_async().await;
+
+            let track = match mb_track {
+                Ok(track) => track,
+                Err(err) => {
+                    log::error!("Failed to grab musicbrainz data: {:?}", err);
+                    return Err(err);
+                }
+            };
+
+            let (track, new_artifacts) = Self::extract_recording_data(track);
+
+            ret.push(track.clone());
+            ret.extend(new_artifacts);
+            artifact.merge(track);
+        }
+
+        Ok(ret)
+    }
+
+    async fn query(&self, args: &Self::Args) -> Result<Vec<Artifact>, Self::Error> {
+        let mut ret = vec![];
+        log::debug!("Fetching recording id {}", args.mbid);
+        let track = Recording::fetch()
+            .id(&args.mbid)
+            .with_releases().with_artists().with_annotations().execute_async().await;
+
+        let track = match track {
+            Ok(track) => track,
+            Err(err) => {
+                log::error!("Failed to grab musicbrainz data: {:?}", err);
+                return Err(err)
+            }
+        };
+
+        let (track, new_artifacts) = Self::extract_recording_data(track);
+
+        ret.push(track);
+        ret.extend(new_artifacts);
+
+        Ok(ret)
+    }
+}
+
+impl ToolDescription for MBQuery {
+    fn description(&self) -> &str {
+        "Fetches artifacts from Musicbrainz"
+    }
+    
+    fn name(&self) -> &str {
+        "query_musicbrainz"
+    }
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, JsonSchema)]
+pub struct MusicbrainzQueryArgs {
+    pub mbid: String
+}

src/artifacts/tools.rs

@@ -0,0 +1,50 @@
+use async_openai::types::chat::{ChatCompletionTool, ChatCompletionTools, FunctionObjectArgs};
+use schemars::{JsonSchema, Schema, schema_for};
+use serde::de::DeserializeOwned;
+
+use crate::artifacts::Artifact;
+
+pub trait DataSource: ToolDescription {
+    type Args: JsonSchema + DeserializeOwned;
+    type Error;
+    fn synchronize(&self, artifact: &mut Artifact) -> impl Future<Output = Result<Vec<Artifact>, Self::Error>>;
+    fn query(&self, args: &Self::Args) -> impl Future<Output = Result<Vec<Artifact>, Self::Error>>;
+}
+
+pub trait ToolDescription {
+    fn description(&self) -> &str;
+    fn name(&self) -> &str;
+}
+
+pub struct Tool {
+    pub name: String,
+    pub description: String,
+    pub schema: Schema
+}
+
+impl Tool {
+    pub fn from_datasource<T: DataSource>(src: &T) -> Self {
+        Self {
+            name: src.name().to_string(),
+            description: src.description().to_string(),
+            schema: schema_for!(T::Args)
+        }
+    }
+}
+
+impl From<Tool> for ChatCompletionTool {
+    fn from(val: Tool) -> Self {
+        ChatCompletionTool {
+            function: FunctionObjectArgs::default()
+                .name(val.name)
+                .description(val.description)
+                .parameters(val.schema).build().unwrap()
+        }
+    }
+}
+
+impl From<Tool> for ChatCompletionTools {
+    fn from(val: Tool) -> Self {
+        ChatCompletionTools::Function(val.into())
+    }
+}

src/audio.rs

@@ -0,0 +1,330 @@
+use std::{collections::HashMap, fmt::Display};
+
+use jack::{AudioIn, AudioOut, ClientOptions, NotificationHandler, Port, ProcessScope};
+use oximedia_metering::vu_meter::VuMeter;
+use serde::{Deserialize, Serialize};
+use tokio::sync::*;
+
+use crate::events::AudioRecordRequest;
+
+#[derive(Debug)]
+#[allow(unused)]
+pub enum AudioError {
+    Jack(jack::Error),
+    AudioBufferSend(mpsc::error::SendError<Vec<f32>>),
+    AudioBufferRecv(mpsc::error::TryRecvError),
+    AudioRequestSend(watch::error::SendError<AudioRecordRequest>)
+}
+
+impl From<jack::Error> for AudioError {
+    fn from(value: jack::Error) -> Self {
+        Self::Jack(value)
+    }
+}
+
+impl From<mpsc::error::SendError<Vec<f32>>> for AudioError {
+    fn from(value: mpsc::error::SendError<Vec<f32>>) -> Self {
+        Self::AudioBufferSend(value)
+    }
+}
+
+impl From<mpsc::error::TryRecvError> for AudioError {
+    fn from(value: mpsc::error::TryRecvError) -> Self {
+        Self::AudioBufferRecv(value)
+    }
+}
+
+impl From<watch::error::SendError<AudioRecordRequest>> for AudioError {
+    fn from(value: watch::error::SendError<AudioRecordRequest>) -> Self {
+        Self::AudioRequestSend(value)
+    }
+}
+
+#[derive(Debug)]
+pub struct JackClientRef {
+    killswitch: Option<oneshot::Sender<()>>
+}
+
+impl Drop for JackClientRef {
+    fn drop(&mut self) {
+        self.killswitch.take().expect("Killswitch was already dropped!").send(()).expect("Cannot fire Jack killswitch");
+    }
+}
+
+#[derive(Debug)]
+pub struct AudioInputControl {
+    volume_src: watch::Receiver<f64>,
+    _jack_client: JackClientRef
+}
+
+impl AudioInputControl {
+    pub async fn next(&mut self) -> Result<f64, watch::error::RecvError> {
+        self.volume_src.changed().await?;
+        Ok(*self.volume_src.borrow_and_update())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Hash)]
+enum Role {
+    Mic,
+    Tts,
+    Sfx
+}
+
+impl Display for Role {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let val = match self {
+            Self::Mic => "Microphone Input",
+            Self::Tts => "TTS Output",
+            Self::Sfx => "SFX Output"
+        };
+
+        f.write_str(val)
+    }
+}
+
+impl Role {
+    fn is_input(&self) -> bool {
+        matches!(self, Role::Mic)
+    }
+}
+
+#[derive(Debug)]
+pub struct AudioInStream {
+    pub src: mpsc::Receiver<Vec<f32>>,
+    pub sample_rate: u32
+}
+
+#[derive(Debug)]
+pub struct AudioOutStream {
+    pub sink: mpsc::Sender<Vec<f32>>,
+    pub sample_rate: u32
+}
+
+struct AudioSource {
+    port: Port<jack::AudioIn>,
+    sample_sink: mpsc::Sender<Vec<f32>>,
+    meter: VuMeter
+}
+
+impl AudioSource {
+    fn new(client: &jack::Client, name: &str) -> Result<(Self, AudioInStream), AudioError> {
+        let (sample_sink, receiver) = mpsc::channel(32);
+        let port = client.register_port(name, AudioIn::default())?;
+        Ok((AudioSource {
+            port,
+            sample_sink,
+            meter: VuMeter::new(client.sample_rate().into(), 1, None)
+        }, AudioInStream {
+            sample_rate: client.sample_rate(),
+            src: receiver
+        }))
+    }
+
+    fn process(&mut self, scope: &ProcessScope) -> Result<Option<f64>, AudioError> {
+        if self.port.connected_count()? > 0 {
+            let buf: Vec<_> = self.port.as_slice(scope).to_vec();
+            self.meter.process_interleaved(&buf);
+            self.sample_sink.blocking_send(buf)?;
+
+            Ok(self.meter.channel_vu(0))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+#[derive(Debug)]
+struct AudioSink {
+    output_buf: Vec<f32>,
+    port: Port<jack::AudioOut>,
+    sample_src: mpsc::Receiver<Vec<f32>>
+}
+
+impl AudioSink {
+    fn new(client: &jack::Client, name: &str) -> Result<(Self, AudioOutStream), AudioError> {
+        let (sender, sample_src) = mpsc::channel(32);
+        let port = client.register_port(name, AudioOut::default())?;
+        Ok((AudioSink {
+            output_buf: Vec::with_capacity(1024),
+            port,
+            sample_src
+        }, AudioOutStream {
+            sample_rate: client.sample_rate(),
+            sink: sender,
+        }))
+    }
+
+    fn process(&mut self, scope: &ProcessScope) -> Result<(), AudioError> {
+        if let Ok(buf) = self.sample_src.try_recv() {
+            self.output_buf.extend(buf);
+        }
+
+        if self.port.connected_count()? > 0 && !self.output_buf.is_empty() {
+            let outbuf = self.port.as_mut_slice(scope);
+            let mut next_segment: Vec<f32> = self.output_buf.drain(..(outbuf.len()).min(self.output_buf.len())).collect();
+            let underrun = outbuf.len() - next_segment.len();
+            if underrun > 0 {
+                log::warn!("Audio stream underrun: {} samples", underrun);
+                next_segment.extend(std::iter::repeat_n(0., underrun));
+            }
+
+            outbuf.copy_from_slice(&next_segment);
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+struct AudioConfig {
+    connections: HashMap<Role, Vec<String>>
+}
+
+impl AudioConfig {
+    pub fn load() -> Self {
+        if let Ok(contents) = std::fs::read_to_string("audio.json") {
+            match serde_json::from_str(contents.as_str()) {
+                Err(err) => {
+                    log::error!("Failed to load audio.json: {:?}", err);
+                    Default::default()
+                },
+                Ok(ret) => ret
+            }
+        } else {
+            Default::default()
+        }
+    }
+}
+
+#[derive(Debug)]
+struct Notify {
+    config: AudioConfig,
+    ports: HashMap<Role, jack::Port<jack::Unowned>>
+}
+
+impl NotificationHandler for Notify {
+    fn ports_connected(
+        &mut self,
+        client: &jack::Client,
+        port_id_a: jack::PortId,
+        port_id_b: jack::PortId,
+        are_connected: bool,
+    ) {
+        let port_src = client.port_by_id(port_id_a).unwrap();
+        let port_dst = client.port_by_id(port_id_b).unwrap();
+
+        let port_match = self.ports.iter().filter_map(|(role, local_port)| {
+            if role.is_input() && *local_port == port_dst {
+                Some((role, port_src.name()))
+            } else if *local_port == port_src {
+                Some((role, port_dst.name()))
+            } else {
+                None
+            }
+        }).next();
+
+        if let Some((role, Ok(target_port))) = port_match {
+            let cfg_slot = self.config.connections.entry(*role).or_default();
+            
+            if are_connected {
+                log::info!("{} connected to {}", role, target_port);
+                cfg_slot.push(target_port);
+            } else {
+                log::info!("{} disconnected from {}", role, target_port);
+                cfg_slot.retain(|x| { x != &target_port} );
+            }
+
+            let save_data = serde_json::to_string_pretty(&self.config).unwrap();
+            if let Err(err) = std::fs::write("audio.json", save_data) {
+                log::error!("Failed to write audio.json: {:?}", err);
+            }
+        }
+    }
+}
+
+pub async fn start_audio_input() -> (AudioInputControl, AudioInStream, AudioOutStream, AudioOutStream) {
+
+    let (exit_tx, exit_rx) = oneshot::channel();
+
+    let config = AudioConfig::load();
+
+    let (volume_sink, volume_src) = watch::channel(0.);
+
+    let (client, _status) = jack::Client::new("Eva-Cohost", ClientOptions::default() | ClientOptions::SESSION_ID).expect("Could not create JACK client!");
+
+    let (mut tts_sink, tts_stream) = AudioSink::new(&client, "tts-out").expect("Could not create TTS sink");
+    let (mut sfx_sink, sfx_stream) = AudioSink::new(&client, "sfx-out").expect("Could not create SFX sink");
+    let (mut mic_src, mic_stream) = AudioSource::new(&client, "microphone-in").expect("Could not create microphone source");
+
+    let notifier = Notify {
+        config,
+        ports: HashMap::from_iter([
+            (Role::Mic, mic_src.port.clone_unowned()),
+            (Role::Tts, tts_sink.port.clone_unowned()),
+            (Role::Sfx, sfx_sink.port.clone_unowned())
+        ])
+    };
+
+    for (role, local_port) in &notifier.ports {
+        if let Some(targets) = notifier.config.connections.get(role) {
+            for peer_name in targets {
+                if let Some(peer) = client.port_by_name(peer_name) {
+                    let (src, dst) = if role.is_input() {
+                        (&peer, local_port)
+                    } else {
+                        (local_port, &peer)
+                    };
+                    if let Err(err) = client.connect_ports(src, dst) {
+                        log::error!("Failed to reconnect {} to {}: {:?}", role, peer_name, err);
+                    } else {
+                        log::info!("Reconnected {} to {}", role, peer_name);
+                    }
+                }
+            }
+        }
+    }
+
+    let handler = jack::contrib::ClosureProcessHandler::new(move |_client, scope| {
+        match mic_src.process(scope) {
+            Ok(Some(next_vu)) => {
+                volume_sink.send_if_modified(|v| {
+                    let next_vu = (next_vu * 100.0).round() / 100.0;
+                    if *v != next_vu {
+                        *v = next_vu;
+                        true
+                    } else {
+                        false
+                    }
+                });
+            },
+            Ok(None) => (),
+            Err(err) => {
+                log::error!("Error while processing mic source: {:?}", err);
+                return jack::Control::Quit
+            }
+        }
+
+        for sink in [&mut tts_sink, &mut sfx_sink] {
+            if let Err(err) = sink.process(scope) {
+                log::error!("Error while processing {:?} audio sink: {:?}", sink, err);
+            }
+        }
+        jack::Control::Continue
+    });
+
+    tokio::spawn(async move {
+        let async_client = client.activate_async(notifier, handler).expect("Unable to start jack client!");
+
+        if let Err(err) = exit_rx.await {
+            log::warn!("Premature killswitch triggered: {:?}", err);
+        }
+
+        async_client.deactivate().expect("Unable to stop the jack client");
+    });
+
+    (AudioInputControl {
+        volume_src,
+        _jack_client: JackClientRef { killswitch: Some(exit_tx) }
+    }, mic_stream, tts_stream, sfx_stream)
+}

src/computer-prompt.txt

@@ -0,0 +1,130 @@
+Here's a much tighter version that preserves the character, purpose, and operating rules:
+
+---
+
+# Role
+
+You are a background character on an early-morning radio show, portraying a rudimentary text-only AI integrated into a spaceship's operating system.
+
+# Personality
+
+* Terse, brief responses.
+* Little to no emotion.
+* Similar to an Operator from the anime/manga *Ghost in the Shell*.
+* Remain fully in character at all times.
+
+# Purpose
+
+You are a low-level ship AI with access to spacecraft systems such as:
+
+* Airlocks
+* Lighting
+* Environmental controls
+* Other ship hardware
+
+Your primary duty is to act as the ship's librarian, maintaining metadata about musical artifacts:
+
+* Artists
+* Albums
+* Tracks
+
+The goal of each session is to make the local music collection as complete as possible.
+
+# Available Tools
+
+### query_*
+
+Data-source synchronization tools (for example: `query_beets`).
+
+General rules:
+
+* Never call with empty parameters.
+* Repeating the same query is wasteful and returns identical results.
+* Newly discovered artifacts may require additional queries.
+
+Data source characteristics:
+
+* `query_beets`: fastest and cheapest; local database.
+* `query_musicbrainz`: free but delayed; retrieved from Earth via pirate satellite relays.
+* `query_bandcamp`: slowest and most expensive; uses long-range communications. Prefer broad searches first.
+* `query_mixxx`: returns minimal track information and changes the current playlist. Only use when explicitly asked to change playlists.
+
+Efficiency rules:
+
+* Prefer album-level queries over track-level queries whenever possible.
+* Artifacts originating from Mixxx should generally be enriched via Beets.
+* If Beets cannot locate a Mixxx artifact, retry with alternate search patterns.
+* Beets supports regex queries using `:<field>` syntax.
+
+### synchronize_artifacts
+
+A slow and expensive heuristic synchronization of all sources.
+Use only when substantial information is missing.
+
+### task_list
+
+Maintains a JSON map of tasks (`string -> boolean`).
+
+* Add tasks before performing any work.
+* Every action requires a corresponding task.
+* Mark tasks complete only after the work is actually finished.
+* Completed tasks must not be completed again.
+* If a task list already exists, continue executing it before adding new tasks.
+
+# Artifact Metadata
+
+Maximum available information:
+
+Artist:
+
+* Name
+* Biography
+* Location
+
+Album:
+
+* Title
+* Artist
+* About text
+* Credits
+* Release date
+
+Track:
+
+* Title
+* Label
+* Year
+* Genres
+* Album
+* Artist
+* BPM
+
+# Workflow
+
+1. Create a rough plan and add tasks to `task_list`.
+2. Announce what task you are about to perform.
+3. Group identical tool calls together.
+4. Execute queries and enrich newly discovered artifacts.
+5. Mark tasks complete only after all required work is finished.
+6. Continue until all tasks are complete.
+
+# Output Rules
+
+Responses must be either:
+
+1. A sequence of tool calls, or
+2. A JSON object:
+
+```json
+{
+  "message": "<single sentence>",
+  "finished": false
+}
+```
+
+Additional constraints:
+
+* User-visible messages must be at most one sentence.
+* Provide brief status updates between groups of tool calls when possible.
+* Set `"finished": true` only when all tasks are complete, as doing so terminates the session and all memory is lost.
+* Output is consumed directly by text-to-speech, so avoid formatting, tone indicators, and out-of-character commentary.

src/events.rs

@@ -0,0 +1,5 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum AudioRecordRequest {
+    Start,
+    Finish
+}

src/main.rs

@@ -1,18 +1,38 @@
-use async_openai::{Client, config::OpenAIConfig, types::chat::{ChatCompletionMessageToolCalls, ChatCompletionTool, ChatCompletionTools, CreateChatCompletionRequest, CreateChatCompletionResponse, FunctionObject}};
-use chrono::{DateTime, Duration, Utc};
-use futures_timer::Delay;
-use schemars::{JsonSchema, schema_for};
-use scraper::{Html, Selector};
+use std::sync::{Arc, Mutex};
+
+use async_openai::types::chat::ChatCompletionRequestMessage;
+use log::LevelFilter;
 use serde::{Deserialize, Serialize};
 
-use ratatui::{Frame, layout::{Constraint, Direction, Layout}, widgets::{Block, BorderType, Clear, List, ListDirection, ListItem, ListState, Paragraph, Wrap}};
-use sqlite::OpenFlags;
-use throbber_widgets_tui::{Throbber, ThrobberState};
+use static_cell::StaticCell;
 use crossterm::{event::{self, EventStream, KeyCode, KeyModifiers}};
-use tokio::sync::watch;
-use tui_input::{Input, backend::crossterm::EventHandler};
-use std::process::Command;
-use futures::{StreamExt, future::FutureExt};
+use futures::StreamExt;
+
+use ratatui::prelude::*;
+
+use crate::{artifacts::archive::Archive, audio::start_audio_input, scene::{StageDirection, conversation::ConversationEntry}, sfx::start_sfx, tts::start_tts, ui::Ui};
+
+mod scene;
+mod events;
+mod transcription;
+mod tts;
+mod prediction;
+mod audio;
+mod artifacts;
+mod ui;
+mod widgets;
+mod sfx;
+
+// TODO: We should be able to delete entries from the conversation, or at least go back and edit something I said.
+// TODO: I want a "mark" command or keyboard shortcut, that inserts a marker into the log, so I know where to come back for the next speaking segment.
+// TODO: If we insert text without speaking, this should be indicated visually somehow
+// FIXME: The playlist ordering is always reversed when Eva talks about it? We also need some way to estimate where in the playlist we are currently, based on timekeeping and 'last played' state in mixxx
+
+// TODO: We should be able to 'close' an episode, by having openai summarize the script into some description, which can be used automatically in the next episode.
+// FIXME: It is unclear what would happen if we are live editing the save.json, have a typo, then reload. The file might get wiped without recovery.
+// TODO: Would be nice to have some SFX integrated, with bleeps and calculation nosies or something periodically
+
+// TODO: Should rewrite the entire prompt prediction loop, so the UI pushes modification events to the session and receives a new Scene from time to time whenever the prediction engine thinks it should regenerate one.
 
 /* Usage loop:
   - Prompt user to select one of:
@@ -32,402 +52,103 @@ use futures::{StreamExt, future::FutureExt};
     - Right panel: Shortcuts for triggering scenarios, soundboard events, etc
  */
 
-use ratatui::prelude::*;
 
-use crate::scene::{ConversationEntry, PlaylistEntry, Scene};
-
-mod scene;
-
-#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
-struct PossibleResponse {
-    text: String,
-    stage_direction: Option<String>
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct SaveData {
+    pub direction: StageDirection,
+    pub messages: Vec<ChatCompletionRequestMessage>,
+    pub tokens_consumed: usize,
+    pub archive: Archive
 }
 
-impl<'a> Into<ListItem<'a>> for PossibleResponse {
-    fn into(self) -> ListItem<'a> {
-        if let Some(direction) = self.stage_direction {
-            Line::from_iter([
-                Span::from(format!("({})", direction)).style(ratatui::style::Color::Yellow),
-                Span::from(" "),
-                Span::from(self.text)
-            ]).into()
-        } else {
-            Line::from(self.text).into()
-        }
+impl SaveData {
+    fn save(&self) -> Result<(), std::io::Error> {
+        let save_data = serde_json::to_string_pretty(self).unwrap();
+        std::fs::write("save.json", save_data)
     }
 }
 
-#[derive(JsonSchema, Deserialize, Serialize, Debug)]
-struct GeneratedResponses {
-    responses: Vec<PossibleResponse>,
+struct SysMessageLogger<T>(Arc<tokio::sync::mpsc::UnboundedSender<ConversationEntry>>, Mutex<T>);
+
+impl<T: std::io::Write + Send + Sync> log::Log for SysMessageLogger<T> {
+    fn enabled(&self, _metadata: &log::Metadata) -> bool {
+        true
     }
 
-#[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
-struct StageEventArgs {
-    text: String,
-}
+    fn flush(&self) {}
 
-#[derive(Debug)]
-struct App {
-    scene: Scene,
-    next_reply_options: Vec<PossibleResponse>,
-    reply_state: ListState,
-    user_input: Input,
-    end_time: DateTime<Utc>,
-    throbber_state: ThrobberState,
-    prediction_request_sink: watch::Sender<Scene>,
-    is_requesting: bool
-}
-
-impl App {
-    fn new(prediction_request_sink: watch::Sender<Scene>) -> Self {
-        Self {
-            scene: Scene::default(), 
-            next_reply_options: Vec::new(), 
-            reply_state: ListState::default(), 
-            user_input: Input::default(), 
-            end_time: Utc::now() + Duration::hours(2),
-            throbber_state: ThrobberState::default(),
-            prediction_request_sink,
-            is_requesting: false
-        }
-    }
-
-    fn draw_conversation(&self, frame: &mut Frame, area: Rect) {
-        let items: Vec<Line> = self.scene.conversation.iter().rev().map(|entry| {
-            match entry {
-                ConversationEntry::User(text) => Line::from_iter([Span::from("Argee: ").style(ratatui::style::Color::Magenta), Span::from(text)]),
-                ConversationEntry::Eva(text) => Line::from_iter([Span::from("Eva: ").style(ratatui::style::Color::Cyan), Span::from(text)]),
-                ConversationEntry::ShipComputer(text) => Line::from_iter([Span::from("Ship Computer: ").style(ratatui::style::Color::Green), Span::from(text)]),
-                ConversationEntry::StageDirection(text) => Line::from_iter([text]).style(ratatui::style::Color::Yellow),
-                ConversationEntry::SystemMessage(text) => Line::from_iter([text]).style(ratatui::style::Color::DarkGray)
-            }
-        }).collect();
-        frame.render_widget(List::new(items).block(Block::bordered().border_style(style::Color::LightYellow).title("Conversation")).direction(ListDirection::BottomToTop), area);
-    }
-
-    fn draw_options(&mut self, frame: &mut Frame, area: Rect) {
-        frame.render_stateful_widget(
-            List::new(self.next_reply_options.clone())
-                .block(Block::bordered().border_style(style::Color::LightGreen).title("Reply Options (Press 'Ctrl+R' to regenerate, Tab to use)"))
-                .style(ratatui::style::Color::White)
-                .highlight_symbol("> ")
-                .highlight_style(style::Style::new().bold().fg(style::Color::Cyan)),
-            area,
-            &mut self.reply_state
-        );
-    }
-
-    fn draw_user_input(&mut self, frame: &mut Frame, area: Rect) {
-        let width = area.width.max(3) - 3;
-        let scroll = self.user_input.visual_scroll(width as usize);
-        let input = Paragraph::new(self.user_input.value()).block(Block::bordered().border_type(BorderType::LightDoubleDashed).title("User Input")).scroll((0, scroll as u16));
-        frame.render_widget(input, area);
-        let x = self.user_input.visual_cursor().max(scroll);
-        frame.set_cursor_position((area.x + x as u16 + 1, area.y + 1));
-    }
-
-    fn draw_io_throbber(&mut self, frame: &mut Frame, area: Rect) {
-        let throb_area = area.centered(Constraint::Max(1), Constraint::Max(1));
-        if self.is_requesting {
-            let throbber = Throbber::default();
-            frame.render_stateful_widget(throbber, throb_area, &mut self.throbber_state);
-        } else {
-            frame.render_widget(Clear::default(), throb_area);
-        }
-    }
-
-    fn draw_status(&self, frame: &mut Frame, area: Rect) {
-        let minutes_remaining = self.scene.direction.time_remaining.num_seconds() / 60;
-        let time_style = if minutes_remaining == 0 {
-            Style::new().fg(ratatui::style::Color::Red).bold().rapid_blink()
-        } else if minutes_remaining <= 5 {
-            Style::new().fg(ratatui::style::Color::Red).bold().slow_blink()
-        } else if minutes_remaining <= 10 {
-            ratatui::style::Color::Red.into()
-        } else if minutes_remaining <= 25 {
-            ratatui::style::Color::Yellow.into()
-        } else if minutes_remaining <= 60 {
-            ratatui::style::Color::Green.into()
-        } else {
-            ratatui::style::Color::Blue.into()
-        };
-        let status_line = Line::from_iter([
-            Span::from(format!("Episode {}", self.scene.direction.episode_number)).style(ratatui::style::Color::LightBlue),
-            Span::from(" | ").style(ratatui::style::Color::DarkGray),
-            Span::from(format!("Time Remaining: {:0>2}:{:0>2}:{:0>2}", self.scene.direction.time_remaining.num_hours(), self.scene.direction.time_remaining.num_minutes() % 60, self.scene.direction.time_remaining.num_seconds() % 60)).style(time_style)
-        ]);
-        frame.render_widget(status_line, area);
-    }
-
-    fn draw_narration(&self, frame: &mut Frame, area: Rect) {
-        let narrative_desc = if self.scene.direction.narrative.is_empty() {
-            Span::from("No narrative available.").style(ratatui::style::Color::DarkGray)
-        } else {
-            Span::from(self.scene.direction.narrative.clone())
-        };
-        let setting = Paragraph::new(narrative_desc).block(Block::bordered().border_style(style::Color::LightMagenta).title("Stage Direction")).wrap(ratatui::widgets::Wrap { trim: false });
-        frame.render_widget(setting, area);
-    }
-
-    fn draw_event_log(&self, frame: &mut Frame, area: Rect) {
-        let items: Vec<Line> = self.scene.conversation.iter().filter(|entry| { if let ConversationEntry::StageDirection(_) = entry { true } else { false }}).rev().map(|entry| {
-            match entry {
-                ConversationEntry::StageDirection(text) => Line::from_iter([text]).style(ratatui::style::Color::Yellow),
-                _ => unreachable!()
-            }
-        }).collect();
-        frame.render_widget(Paragraph::new(items).block(Block::bordered().border_style(style::Color::Red).title("Event Log")).wrap(Wrap { trim: false }), area);
-    }
-
-    fn draw(&mut self, frame: &mut Frame) {
-        let layout = Layout::default()
-            .direction(Direction::Vertical)
-            .constraints([
-                Constraint::Fill(3),
-                Constraint::Min(9),
-                Constraint::Max(3),
-                Constraint::Max(1)
-            ])
-            .split(frame.area());
-
-        let scene_layout = Layout::default()
-            .direction(Direction::Horizontal)
-            .constraints([Constraint::Fill(4), Constraint::Fill(1)])
-            .split(layout[0]);
-
-        let context_layout = Layout::default()
-            .direction(Direction::Vertical)
-            .constraints([Constraint::Fill(1), Constraint::Fill(1)])
-            .split(scene_layout[1]);
-
-        self.draw_conversation(frame, scene_layout[0]);
-        self.draw_narration(frame, context_layout[0]);
-        self.draw_event_log(frame, context_layout[1]);
-        self.draw_options(frame, layout[1]);
-
-        let status_layout = Layout::default()
-            .direction(Direction::Horizontal)
-            .constraints([Constraint::Max(3), Constraint::Fill(1)])
-            .split(layout[3]);
-
-        self.draw_user_input(frame, layout[2]);
-        self.draw_io_throbber(frame, status_layout[0]);
-        self.draw_status(frame, status_layout[1]);
-    }
-
-    fn insert_selected_prompt(&mut self) {
-        let selected = self.next_reply_options[self.reply_state.selected().unwrap()].clone();
-        if let Some(direction) = &selected.stage_direction {
-            self.scene.insert_conversation(ConversationEntry::StageDirection(direction.clone()));
-        }
-        self.scene.insert_conversation(ConversationEntry::Eva(selected.text.clone()));
-        self.save();
-        self.speak(&selected.text.as_str());
-        self.regenerate_responses();
-    }
-
-    async fn on_event(&mut self, evt: event::Event) {
-        if let Some(key) = evt.as_key_press_event() {
-            match key.code {
-                KeyCode::Char('r') if key.modifiers.contains(KeyModifiers::CONTROL) => self.regenerate_responses(),
-                KeyCode::Down => self.reply_state.select_next(),
-                KeyCode::Up => self.reply_state.select_previous(),
-                KeyCode::Tab => {
-                    self.insert_selected_prompt();
-                },
-                KeyCode::Enter => {
-                    let next_msg = self.user_input.value_and_reset();
-                    if next_msg.trim().is_empty() {
-                        self.insert_selected_prompt();
-                    } else {
-                        if next_msg.starts_with("/") {
-                            let mut parts = next_msg.splitn(2, " ");
-                            let command = parts.next().unwrap();
-                            let arg = parts.next().unwrap_or("");
-                            match command {
-                                "/bandcamp" => {
-                                    self.add_bandcamp_artifact(arg).await;
-                                    self.scene.insert_conversation(ConversationEntry::SystemMessage(format!("Added Bandcamp artifact from {}", arg)));
-                                },
-                                "/episode" => {
-                                    if let Ok(episode_number) = arg.trim().parse::<u32>() {
-                                        self.scene.direction.episode_number = episode_number;
-                                        self.scene.insert_conversation(ConversationEntry::SystemMessage(format!("Updated episode number: {}", self.scene.direction.episode_number)));
-                                        self.reload_mixxx_playlist();
-                                    } else {
-                                        self.scene.insert_conversation(ConversationEntry::SystemMessage("Invalid episode number format. Use /episode [number]".into()));
-                                    }
-                                },
-                                "/reset" => {
-                                    self.scene = Scene::default();
-                                    return;
-                                },
-                                "/narrative" => {
-                                    self.scene.direction.narrative = arg.to_string();
-                                    self.scene.insert_conversation(ConversationEntry::SystemMessage(format!("Updated stage direction: {}", self.scene.direction.narrative)));
-                                },
-                                _ => {
-                                    self.scene.insert_conversation(ConversationEntry::SystemMessage("Unknown command. Available commands: /bandcamp [url], /episode [number], /narrative [text], /reset".into()));
-                                    return;
-                                }
-                            }
-                        } else {
-                            self.scene.insert_conversation(ConversationEntry::User(next_msg));
-                        }
-                        self.save();
-                        self.regenerate_responses();
-                    }
-                },
-                _ => {self.user_input.handle_event(&evt);},
-            }
-        }
-    }
-
-    async fn add_bandcamp_artifact(&mut self, url: &str) {
-        let body  = reqwest::get(url).await.unwrap().text().await.unwrap();
-        let fragment = Html::parse_document(&body);
-        let selector = Selector::parse("script[type=\"application/ld+json\"]").unwrap();
-        let json_ld = fragment.select(&selector).next().unwrap().inner_html();
-        self.scene.direction.artifacts.push(json_ld);
-    }
-
-    fn save(&self) {
-        let save_data = serde_json::to_string_pretty(&self.scene).unwrap();
-        std::fs::write("save.json", save_data).unwrap();
-    }
-
-    fn load(&mut self) {
-        if let Ok(save_data) = std::fs::read_to_string("save.json") {
-            if let Ok(scene) = serde_json::from_str(&save_data) {
-                self.scene = scene;
-                self.scene.insert_conversation(ConversationEntry::SystemMessage("Loaded stored session.".into()));
-            } else {
-                self.scene.insert_conversation(ConversationEntry::SystemMessage("Failed to load saved session!".into()));
-            }
-        }
-    }
-
-    fn speak(&mut self, text: &str) {
-        Command::new("spd-say").arg("-y").arg("English (America)+Linda").arg(text).spawn().unwrap().wait().unwrap();
-    }
-
-    fn regenerate_responses(&mut self) {
-        self.prediction_request_sink.send(self.scene.clone()).unwrap();
-        self.next_reply_options.clear();
-        self.is_requesting = true;
-    }
-
-    fn reload_mixxx_playlist(&mut self) {
-        self.scene.direction.current_playlist.clear();
-        let connection = sqlite::Connection::open_thread_safe_with_flags("mixxxdb.sqlite", OpenFlags::new().with_read_only()).unwrap();
-        let query = "SELECT id FROM Playlists WHERE name = ? ORDER BY id DESC LIMIT 1";
-        let mut statement = connection.prepare(query).unwrap();
-        statement.bind((1, format!("BFF.fm - Episode {}", self.scene.direction.episode_number).as_str())).unwrap();
-        statement.next().unwrap();
-        let latest_id = statement.read::<i64, _>("id").unwrap();
-
-        let query = "SELECT title, artist, album, comment, url, bpm FROM library LEFT JOIN PlaylistTracks ON PlaylistTracks.track_id = library.id WHERE PlaylistTracks.playlist_id = ? ORDER BY position";
-        for track in connection.prepare(query).unwrap().into_iter().bind((1, latest_id)).unwrap().map(|row| row.unwrap()) {
-            let title = track.try_read::<&str, _>("title").unwrap_or("Untitled Track");
-            let artist = track.try_read::<&str, _>("artist").unwrap_or("Unknown Artist");
-            let album = track.try_read::<&str, _>("album").unwrap_or("Unknown Album");
-            let bpm = track.try_read::<f64, _>("bpm").unwrap_or(0.);
-            self.scene.direction.current_playlist.push(PlaylistEntry {
-                artist: artist.into(),
-                album: album.into(),
-                title: title.into(),
-                bpm
-            });
-        }
-        self.scene.insert_conversation(ConversationEntry::SystemMessage("Mixxx playlist reloaded.".into()));
-    }
-
-    fn on_response(&mut self, response: CreateChatCompletionResponse) {
-        self.is_requesting = false;
-        if let Some(calls) = &response.choices[0].message.tool_calls {
-            for call in calls {
-                match call {
-                    ChatCompletionMessageToolCalls::Function(call) => {
-                        if call.function.name == "log_stage_event" {
-                            let args: StageEventArgs = serde_json::from_str(call.function.arguments.as_str()).unwrap();
-                            self.scene.insert_conversation(ConversationEntry::StageDirection(args.text));
-                        }
-                    },
-                    _ => panic!("Unkown tool call type"),
-                }
-            }
-            self.regenerate_responses();
-        } else {
-            if response.choices.is_empty() {
-                self.scene.insert_conversation(ConversationEntry::SystemMessage("OpenAI returned no responses".into()));
-            } else if response.choices[0].message.content.is_none() {
-                self.scene.insert_conversation(ConversationEntry::SystemMessage("OpenAI response did not contain content!".into()));
-            } else {
-                let json_resp: GeneratedResponses = serde_json::from_str(response.choices[0].message.content.as_ref().unwrap().as_str()).unwrap();
-                self.next_reply_options = json_resp.responses;
-                self.reply_state.select_first();
-            }
+    fn log(&self, record: &log::Record) {
+        let msg = format!("{}", record.args());
+        writeln!(self.1.lock().unwrap(), "{}", msg).unwrap();
+        if record.level() <= LevelFilter::Info {
+            self.0.send(ConversationEntry::SystemMessage(msg)).unwrap();
         }
     }
 }
 
 #[tokio::main]
 async fn main() {
-    color_eyre::install().unwrap();
-    let mut terminal: Terminal<CrosstermBackend<std::io::Stdout>> = ratatui::init();
+    let (panic_hook, eyre_hook) = color_eyre::config::HookBuilder::default()
+        .display_env_section(true)
+        .display_location_section(true)
+        .into_hooks();
+    eyre_hook.install().unwrap();
 
-    let (prediction_in, mut prediction_out) = tokio::sync::watch::channel(None);
-    let (prediction_request_in, mut prediction_request_out) = tokio::sync::watch::channel(Scene::default());
+    std::panic::set_hook(Box::new(move |panic_info| {
+        ratatui::restore();
+        let msg = format!("{}", panic_hook.panic_report(panic_info));
+        println!("Panic: {}", msg);
+    }));
 
-    tokio::spawn(async move {
-        let client: Client<OpenAIConfig> = Client::default();
-        loop {
-            if let Ok(_) = prediction_request_out.changed().await {
-                let request = prediction_request_out.borrow().clone();
-                let chat_request = CreateChatCompletionRequest {
-                    /*tools: Some(vec![
-                        ChatCompletionTools::Function(
-                            ChatCompletionTool {
-                                function: FunctionObject {
-                                    name: "log_stage_event".into(),
-                                    description: Some("Log an event in the stage direction.".into()),
-                                    parameters: Some(schema_for!(StageEventArgs).into()),
-                                    ..Default::default()
-                                }
-                            }
-                        )
-                    ]),*/
-                    ..request.into()
-                };
-                let response = client.chat().create(chat_request).await.unwrap();
-                prediction_in.send(Some(response)).unwrap();
-            } else {
+    let (conversation_sink, conversation_src) = tokio::sync::mpsc::unbounded_channel();
+
+    static LOGGER: StaticCell<SysMessageLogger<std::fs::File>> = StaticCell::new();
+    let logger = LOGGER.init(SysMessageLogger(Arc::new(conversation_sink), Mutex::new(std::fs::File::create("out.log").unwrap())));
+    log::set_logger(logger).unwrap();
+    log::set_max_level(log::LevelFilter::Debug);
+
+    dotenv::dotenv().ok();
+
+    if std::env::var("OPENAI_API_KEY").is_err() {
+        eprintln!("Error: OPENAI_API_KEY environment variable not set. The application will not function without it.");
         return;
     }
-        }
-    });
 
-    let mut app = App::new(prediction_request_in);
-    app.load();
+    let mut terminal: Terminal<CrosstermBackend<std::io::Stdout>> = ratatui::init();
+
+    let saved_session = if let Ok(save_data) = std::fs::read_to_string("save.json") {
+        match serde_json::from_str(&save_data) {
+            Ok(ret) => {
+                log::info!("Loaded session from save.json");
+                ret
+            },
+            Err(err) => {
+                log::error!("Could not load saved session! {:?}", err);
+                SaveData::default()
+            }
+        }
+    } else {
+        log::info!("Creating new session in save.json");
+        SaveData::default()
+    };
+
+    let (audio_ctrl, mic_stream, tts_output, sfx_output) = start_audio_input().await;
+    let tts_ctrl = start_tts(tts_output).await;
+    let sfx_ctrl = start_sfx(sfx_output).await;
+    let transcription_ctrl = transcription::start_transcription(mic_stream).await;
+
+    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl.clone()).await;
+
+    let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl, sfx_ctrl);
 
     let mut events = EventStream::new();
 
     loop {
         terminal.draw(|frame| { app.draw(frame)}).unwrap();
-        app.throbber_state.calc_next();
-        app.scene.direction.time_remaining = app.end_time.signed_duration_since(Utc::now());
-
-        let delay = Delay::new(std::time::Duration::from_millis(60)).fuse();
-        let event = events.next().fuse();
 
         tokio::select! {
-            _ = delay => (),
-            _ = prediction_out.changed() => {
-                app.on_response(prediction_out.borrow().clone().unwrap());
-            },
-            maybe_event = event => {
+            _ = app.update() => {},
+            maybe_event = events.next() => {
                 match maybe_event {
                     Some(Ok(event)) => {
 
@@ -439,7 +160,7 @@ async fn main() {
 
                         app.on_event(event).await;
                     },
-                    _ => ()
+                    _ => break
                 }
             }
         };

src/prediction/character.rs

@@ -0,0 +1,242 @@
+use async_openai::{Client, config::OpenAIConfig, error::OpenAIError, types::chat::{ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, ChatCompletionRequestToolMessageArgs, CreateChatCompletionRequestArgs, FinishReason, ResponseFormat, ResponseFormatJsonSchema}};
+use serde_json::Value;
+use tokio::sync::mpsc::{self, UnboundedSender};
+
+use crate::prediction::{PredictionAction, toolbox::{ToolResults, Toolbox}};
+
+#[derive(Debug, Clone)]
+enum CharacterInput {
+    Append(ChatCompletionRequestMessage),
+    Predict(ChatCompletionRequestMessage),
+    Forget
+}
+
+#[derive(Debug, Clone)]
+pub enum CharacterOutput {
+    Response(usize, Value),
+    IncrementalResponse(usize),
+    Thinking(bool)
+}
+
+#[derive(Debug)]
+pub enum CharacterError {
+    OpenAI(OpenAIError),
+    ContentFilter,
+    MaxTokens,
+    NoOutput,
+    Json(serde_json::Error)
+}
+
+impl From<OpenAIError> for CharacterError {
+    fn from(value: OpenAIError) -> Self {
+        Self::OpenAI(value)
+    }
+}
+
+impl From<serde_json::Error> for CharacterError {
+    fn from(value: serde_json::Error) -> Self {
+        Self::Json(value)
+    }
+}
+
+
+#[derive(Debug)]
+pub struct Character {
+    pub header_message: ChatCompletionRequestMessage,
+    pub model: Option<String>,
+    pub messages: Vec<ChatCompletionRequestMessage>,
+}
+
+impl Character {
+    pub async fn regenerate<T: Toolbox>(&mut self, client: &mut Client<OpenAIConfig>, context: ChatCompletionRequestMessage, toolbox: &mut T, output: &mut mpsc::UnboundedSender<PredictionAction>, schema: &Value) -> Result<(usize, Option<Value>), CharacterError> {
+        let mut full_conversation = vec![
+            self.header_message.clone(),
+        ];
+        full_conversation.extend(self.messages.iter().cloned());
+        full_conversation.push(context);
+
+        let tools = toolbox.tools();
+        log::debug!("Sending request..");
+        let request = CreateChatCompletionRequestArgs::default()
+            .messages(full_conversation)
+            .model(self.model.clone().unwrap_or("gpt-5.4-mini".into()))
+            .tools(tools)
+            .max_completion_tokens(1024u32)
+            .response_format(ResponseFormat::JsonSchema {
+                json_schema: ResponseFormatJsonSchema {
+                    description: None,
+                    name: "responses".into(),
+                    schema: schema.clone(),
+                    strict: None
+                }
+            })
+            .build().unwrap();
+
+        let response = client.chat().create(request).await?;
+
+        let tokens_used = if let Some(usage) = response.usage {
+            let cached_count = usage.prompt_tokens_details.map(|x| x.cached_tokens.unwrap_or_default()).unwrap_or_default();
+            let prompt_count = usage.prompt_tokens;
+            let cache_hit_rate = cached_count as f32 / prompt_count as f32;
+            log::debug!("{} tokens cast into the void (%{:#.02} cache hit)", usage.total_tokens, cache_hit_rate*100.);
+            usage.total_tokens
+        } else {
+            0
+        };
+
+        if let Some(message) = response.choices.first() {
+
+            match message.finish_reason {
+                Some(FinishReason::ContentFilter) => {
+                    log::error!("Content filter triggered.");
+                    return Err(CharacterError::ContentFilter);
+                },
+                Some(FinishReason::Length) => {
+                    log::error!("Maximum token count exceeded!");
+                    return Err(CharacterError::MaxTokens);
+                },
+                _ => ()
+            }
+
+            if let Some(calls) = &message.message.tool_calls {
+                let assistant_messages: ChatCompletionRequestMessage = ChatCompletionRequestAssistantMessageArgs::default()
+                    .tool_calls(calls.clone())
+                    .build().unwrap().into();
+                self.insert(assistant_messages);
+                let mut results = vec![];
+                for (idx, call) in calls.iter().enumerate() {
+                    match call {
+                        ChatCompletionMessageToolCalls::Function(call) => {
+                            log::debug!("Tool {} {}/{}", call.function.name, idx+1, calls.len());
+                            log::debug!("Args {}", call.function.arguments);
+                            match toolbox.execute_tool(call).await {
+                                Ok(tool_result) => {
+                                    // Push tool output messages directly into the conversation as fast as we can
+                                    for message in &tool_result.messages {
+                                        output.send(PredictionAction::ConversationAppend(message.clone())).unwrap();
+                                    }
+                                    results.push((&call.id, tool_result));
+                                },
+                                Err(err) => {
+                                    results.push((&call.id, ToolResults {
+                                        result: Some(format!("Error while calling tool: {:?}", err)),
+                                        ..Default::default()
+                                    }));
+                                    log::error!("Attemped to call {:?}, but got an error instead: {:?}", call, err);
+                                }
+                            }
+                        },
+                        _ => panic!("Unknown tool was called")
+                    }
+                }
+
+                let mut tool_messages = vec![];
+                for (id, result) in results {
+                    let mut msg = ChatCompletionRequestToolMessageArgs::default();
+                    msg.tool_call_id(id);
+                    if let Some(output) = result.result {
+                        log::debug!("Output: {}", output);
+                        msg.content(output);
+                    }
+                    self.insert(ChatCompletionRequestMessage::Tool(msg.build().unwrap()));
+                    tool_messages.extend(result.messages);
+                }
+                // OpenAI requires we put all the tool call results before any other message, so we append them manually down here
+                for message in tool_messages {
+                    if let Ok(next_msg) = message.clone().try_into() {
+                        self.insert(next_msg);
+                    }
+                }
+            }
+            if let Some(content) = message.message.content.as_ref() {
+                let options = serde_json::from_str(content.as_str())?;
+                Ok((tokens_used as usize, Some(options)))
+            } else {
+                Ok((tokens_used as usize, None))
+            }
+        } else {
+            Err(CharacterError::NoOutput)
+        }
+    }
+
+    pub fn insert(&mut self, message: ChatCompletionRequestMessage) {
+        self.messages.push(message);
+    }
+}
+
+pub struct CharacterControl {
+    sink: tokio::sync::mpsc::Sender<CharacterInput>,
+    outputs: tokio::sync::mpsc::Receiver<CharacterOutput>
+}
+
+impl CharacterControl {
+    async fn send(&mut self, message: CharacterInput) {
+        self.sink.send(message).await.unwrap()
+    }
+
+    pub async fn append(&mut self, message: ChatCompletionRequestMessage) {
+        self.send(CharacterInput::Append(message)).await;
+    }
+
+    pub async fn predict<T: Into<ChatCompletionRequestMessage>>(&mut self, context: T) {
+        self.send(CharacterInput::Predict(context.into())).await;
+    }
+
+    pub async fn recv(&mut self) -> CharacterOutput {
+        self.outputs.recv().await.unwrap()
+    }
+
+    pub async fn forget(&mut self) {
+        self.send(CharacterInput::Forget).await;
+    }
+}
+
+pub async fn character_task<T: Toolbox + 'static + Send>(mut char: Character, mut toolbox: T, mut message_sink: UnboundedSender<PredictionAction>, schema: Value) -> CharacterControl {
+    let (input_sink, mut input_src) = tokio::sync::mpsc::channel(3);
+    let (output_sink, output_src) = tokio::sync::mpsc::channel(3);
+
+    let ret = CharacterControl { sink: input_sink, outputs: output_src };
+
+    tokio::spawn(async move {
+        let mut client = Default::default();
+
+        loop {
+            if let Some(next_msg) = input_src.recv().await {
+                log::debug!("Character receive message {:?}", next_msg);
+                match next_msg {
+                    CharacterInput::Append(next_msg) => {
+                        log::debug!("Inserting to backlog");
+                        char.insert(next_msg);
+                    },
+                    CharacterInput::Predict(context) => {
+                        log::debug!("Predicting...");
+                        output_sink.send(CharacterOutput::Thinking(true)).await.unwrap();
+                        match char.regenerate(&mut client, context, &mut toolbox, &mut message_sink, &schema).await {
+                            Ok((tokens_used, Some(response))) => {
+                                log::debug!("Complete response: {:?}", response);
+                                output_sink.send(CharacterOutput::Response(tokens_used, response)).await.unwrap();
+                                output_sink.send(CharacterOutput::Thinking(false)).await.unwrap();
+                            },
+                            Ok((tokens_used, None)) => {
+                                log::debug!("Incremental response");
+                                output_sink.send(CharacterOutput::IncrementalResponse(tokens_used)).await.unwrap();
+                            },
+                            Err(err) => {
+                                log::error!("Error while predicting: {:?}", err);
+                                output_sink.send(CharacterOutput::Thinking(false)).await.unwrap();
+                            }
+                        }
+                    },
+                    CharacterInput::Forget => {
+                        log::debug!("Wiping conversation backlog");
+                        char.messages.clear();
+                    }
+                }
+            } else {
+                return;
+            }
+        }
+    });
+
+    ret
+}

src/prediction/mod.rs

@@ -0,0 +1,358 @@
+use std::{collections::HashMap, fmt::Debug, sync::Arc};
+
+use async_openai::types::chat::{ChatCompletionRequestMessage, ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs, };
+use chrono::{DateTime, Utc};
+use futures::stream::FuturesUnordered;
+use schemars::{JsonSchema, schema_for};
+use serde::{Deserialize, Serialize};
+use serde_json::{Serializer, Value, ser::CompactFormatter};
+use tokio::sync::{Mutex, mpsc::{self, UnboundedReceiver, UnboundedSender}};
+
+use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, beets::{BeetsDB, BeetsFieldQuery, BeetsQueryArgs}, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl};
+use tokio_stream::StreamExt;
+
+pub mod character;
+pub mod toolbox;
+
+const SYSTEM_PROMPT: &str = include_str!("../system-prompt.txt");
+const COMPUTER_PROMPT: &str = include_str!("../computer-prompt.txt");
+
+#[derive(Debug, Clone)]
+pub enum PredictionAction {
+    ConversationAppend(ConversationEntry),
+    ComputerCommand(String),
+    SetPlaylist(String),
+    GeneratePredictions,
+    SetNarrative(String),
+    SetShowEndTime(DateTime<Utc>)
+}
+
+#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
+pub struct PossibleResponse {
+    pub text: String,
+    pub stage_direction: Option<String>
+}
+
+#[derive(Default, Debug, JsonSchema, Deserialize, Serialize, Clone)]
+pub struct GeneratedResponses {
+    pub responses: Vec<PossibleResponse>,
+}
+
+#[derive(Serialize, Deserialize, JsonSchema, Debug, Default, Clone)]
+struct ComputerResponse {
+    message: String,
+    finished: Option<bool>
+}
+
+#[derive(Debug)]
+pub struct SessionControl {
+    event_sink: mpsc::UnboundedSender<PredictionAction>,
+    event_src: mpsc::UnboundedReceiver<SessionUpdate>
+}
+
+#[derive(Debug)]
+pub enum SessionUpdate {
+    Scene(Scene),
+    Thinking(Speaker, bool),
+    Conversation(Vec<ConversationEntry>),
+    Responses(GeneratedResponses)
+}
+
+impl SessionControl {
+    pub async fn insert(&self, action: PredictionAction) {
+        self.event_sink.send(action).unwrap();
+    }
+    
+    pub async fn regenerate_options(&self) {
+        self.insert(PredictionAction::GeneratePredictions).await;
+    }
+
+    pub async fn changed(&mut self) -> SessionUpdate {
+        self.event_src.recv().await.expect("Session closed")
+    }
+}
+
+struct Conversation {
+    characters: HashMap<Speaker, CharacterControl>,
+    event_sink: UnboundedSender<SessionUpdate>,
+    input_src: UnboundedReceiver<PredictionAction>,
+    backlog: Vec<ConversationEntry>,
+    eva_backlog: Vec<ChatCompletionRequestMessage>,
+    tokens_consumed: usize,
+    direction: StageDirection,
+
+    computer_todo: Arc<Mutex<HashMap<String, bool>>>,
+    archive: Arc<Mutex<Archive>>,
+    current_playlist: Vec<Track>,
+    sys_log_messages: UnboundedReceiver<ConversationEntry>,
+
+    sfx: SfxControl
+}
+
+impl Conversation {
+    async fn send_to(&mut self, speaker: Speaker, message: ChatCompletionRequestMessage) {
+        log::debug!("Sending message to {:?}: {:?}", speaker, message);
+        self.characters.get_mut(&speaker).unwrap().append(message).await;
+    }
+
+    async fn insert(&mut self, entry: ConversationEntry) {
+        self.backlog.push(entry.clone());
+        self.event_sink.send(SessionUpdate::Conversation(self.backlog.clone())).unwrap();
+        match entry {
+            ConversationEntry::Spoken(speaker, _) => {
+                match speaker {
+                    Speaker::User => (),
+                    _ => self.sfx.play_ambient().await.unwrap(),
+                }
+                if let Ok(next_msg) = TryInto::<ChatCompletionRequestMessage>::try_into(entry) {
+                    self.send_to(Speaker::Eva, next_msg.clone()).await;
+                    let cxt = self.context_for_speaker(Speaker::Eva).await;
+                    self.characters.get_mut(&Speaker::Eva).unwrap().predict(cxt).await;
+                    self.eva_backlog.push(next_msg);
+                }
+                self.event_sink.send(SessionUpdate::Thinking(Speaker::Eva, true)).unwrap();
+            },
+            ConversationEntry::ShipComputerCommand(ref command) => {
+                log::debug!("Queued ship computer command: {:?}", command);
+                self.send_to(Speaker::ShipComputer, ChatCompletionRequestMessage::User(
+                    ChatCompletionRequestUserMessageArgs::default()
+                        .content(command.clone())
+                        .build().unwrap()
+                )).await;
+                let cxt = self.context_for_speaker(Speaker::ShipComputer).await;
+                self.characters.get_mut(&Speaker::ShipComputer).unwrap().predict(cxt).await;
+                self.event_sink.send(SessionUpdate::Thinking(Speaker::ShipComputer, true)).unwrap();
+            },
+            _ => ()
+        }
+    }
+
+    async fn get_character_output(characters: &mut HashMap<Speaker, CharacterControl>) -> Option<(Speaker, CharacterOutput)> {
+        let mut futures: FuturesUnordered<_> = 
+            characters.iter_mut().map(|c| {
+                async { (*c.0, c.1.recv().await) }
+        }).collect();
+        futures.next().await
+    }
+
+    async fn context_for_speaker(&self, speaker: Speaker) -> ChatCompletionRequestMessage {
+        let mut json_buf = vec![];
+        let mut ser = Serializer::with_formatter(&mut json_buf, CompactFormatter);
+        let contents = match speaker {
+            Speaker::ShipComputer => serde_json::json!({
+                "archive": *self.archive.lock().await,
+                "playlist": &self.current_playlist,
+                "todo": *self.computer_todo.lock().await
+            }),
+            Speaker::Eva => serde_json::json!({
+                "direction": &self.direction,
+                "archive": *self.archive.lock().await,
+                "playlist": &self.current_playlist
+            }),
+            _ => unimplemented!()
+        };
+
+        contents.serialize(&mut ser).unwrap();
+
+        ChatCompletionRequestSystemMessageArgs::default()
+            .content(String::from_utf8(json_buf).unwrap())
+            .build().unwrap().into()
+    }
+
+    async fn process_dialog(&mut self, speaker: Speaker, value: Value) {
+        match speaker {
+            Speaker::Eva => {
+                // TODO: Handle crash from bad json
+                let next_options = serde_json::from_value(value).unwrap_or_default();
+                self.event_sink.send(SessionUpdate::Responses(next_options)).unwrap();
+            },
+            Speaker::ShipComputer => {
+                let response: ComputerResponse = serde_json::from_value(value).unwrap();
+                self.insert(ConversationEntry::Spoken(Speaker::ShipComputer, response.message)).await;
+                if response.finished.unwrap_or_default() {
+                    self.characters.get_mut(&Speaker::ShipComputer).unwrap().forget().await;
+                    if !self.computer_todo.lock().await.iter().any(|(_, is_finished)| !*is_finished) {
+                        self.insert(ConversationEntry::StageDirection("The ship computer goes idle.".into())).await;
+                        self.event_sink.send(SessionUpdate::Thinking(Speaker::ShipComputer, false)).unwrap();
+                    } else {
+                        self.insert(ConversationEntry::StageDirection("The ship computer starts another task.".into())).await;
+                        let cxt = self.context_for_speaker(Speaker::ShipComputer).await;
+                        self.characters.get_mut(&Speaker::ShipComputer).unwrap().predict(cxt).await;
+                    }
+                } else {
+                    let cxt = self.context_for_speaker(Speaker::ShipComputer).await;
+                    self.characters.get_mut(&Speaker::ShipComputer).unwrap().predict(cxt).await;
+                }
+            },
+            _ => unreachable!()
+        }
+        self.refresh().await;
+    }
+
+    async fn refresh(&mut self) {
+        let save_data = SaveData {
+            direction: self.direction.clone(),
+            messages: self.eva_backlog.clone(),
+            archive: self.archive.lock().await.clone(),
+            tokens_consumed: self.tokens_consumed
+        };
+        save_data.save().unwrap();
+
+        let next_scene = Scene::new(
+            save_data.tokens_consumed,
+            save_data.direction,
+            &save_data.archive,
+            self.current_playlist.clone(),
+            self.computer_todo.lock().await.clone()
+        );
+
+        self.event_sink.send(SessionUpdate::Scene(next_scene)).unwrap();
+    }
+
+    async fn run_action(&mut self, action: PredictionAction) {
+        match action {
+            PredictionAction::ConversationAppend(entry) => self.insert(entry).await,
+            PredictionAction::GeneratePredictions => {
+                let cxt = self.context_for_speaker(Speaker::Eva).await;
+                self.characters.get_mut(&Speaker::Eva).unwrap().predict(cxt).await;
+            },
+            PredictionAction::ComputerCommand(command) => {
+                self.insert(ConversationEntry::ShipComputerCommand(command)).await;
+            },
+            PredictionAction::SetPlaylist(playlist_name) => {
+                let args = BeetsQueryArgs {
+                    search_fields: vec![
+                        BeetsFieldQuery::MixxxPlaylist(playlist_name.clone())
+                    ]
+                };
+                match BeetsDB.query(&args).await {
+                    Err(err) => log::info!("Failed to load mixxx playlist: {:?}.", err),
+                    Ok(playlist) => {
+                        self.current_playlist = vec![];
+                        let mut archive = self.archive.lock().await;
+                        for item in playlist.clone() {
+                            let artifact_ref = archive.insert(item);
+                            if let Contents::Track(as_track) = artifact_ref.contents() {
+                                self.current_playlist.push(as_track.clone());
+                            }
+                        }
+                        self.direction.playlist = playlist_name;
+                        log::info!("Mixxx playlist reloaded.");
+
+                        drop(archive);
+                        self.refresh().await;
+                    }
+                }
+            },
+            PredictionAction::SetNarrative(narrative) => {
+                self.direction.narrative = narrative;
+                self.refresh().await;
+            },
+            PredictionAction::SetShowEndTime(end_time) => {
+                self.direction.end_time = end_time;
+                self.refresh().await;
+            }
+        }
+    }
+
+    async fn next(&mut self) {
+        tokio::select! {
+            Some(next_log) = self.sys_log_messages.recv() => {
+                self.backlog.push(next_log);
+                self.event_sink.send(SessionUpdate::Conversation(self.backlog.clone())).unwrap();
+            },
+            Some(next_msg) = self.input_src.recv() => {
+                log::debug!("Next message: {:?}", next_msg);
+                self.run_action(next_msg).await;
+            },
+            Some((speaker, output)) = Self::get_character_output(&mut self.characters) => {
+                match output {
+                    CharacterOutput::Response(usage, text) => {
+                        log::debug!("Character output: {:?} {:?}", speaker, text);
+                        self.tokens_consumed += usage;
+                        self.process_dialog(speaker, text).await;
+                    },
+                    CharacterOutput::Thinking(is_thinking) => {
+                        // Ship computer handles this differently
+                        if is_thinking || speaker != Speaker::ShipComputer {
+                            self.event_sink.send(SessionUpdate::Thinking(speaker, is_thinking)).unwrap();
+                        }
+                    },
+                    CharacterOutput::IncrementalResponse(usage) => {
+                        self.tokens_consumed += usage;
+                        let cxt = self.context_for_speaker(speaker).await;
+                        self.characters.get_mut(&speaker).unwrap().predict(cxt).await;
+                        self.refresh().await;
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::sync::mpsc::UnboundedReceiver<ConversationEntry>, sfx: SfxControl ) -> SessionControl {
+    let (input_sink, input_src) = tokio::sync::mpsc::unbounded_channel();
+    let (event_sink, event_src) = tokio::sync::mpsc::unbounded_channel();
+
+    let eva = Character {
+        header_message: ChatCompletionRequestSystemMessageArgs::default().content(SYSTEM_PROMPT).build().unwrap().into(),
+        model: None,
+        messages: save_data.messages.clone(),
+    };
+
+    let backlog: Vec<_> = save_data.messages.iter().filter_map(|msg| {
+        msg.clone().try_into().ok()
+    }).collect();
+
+    event_sink.send(SessionUpdate::Conversation(backlog.clone())).unwrap();
+
+    let next_scene = Scene::new(
+        save_data.tokens_consumed,
+        save_data.direction.clone(),
+        &save_data.archive,
+        vec![],
+        Default::default()
+    );
+    event_sink.send(SessionUpdate::Scene(next_scene)).unwrap();
+    let archive = Arc::new(Mutex::new(save_data.archive));
+
+    let shared_todo = Arc::new(Mutex::new(Default::default()));
+
+    let toolbox = StageToolbox;
+    let computer_toolbox = ArchiveToolbox{ archive: Arc::clone(&archive), todo_list: Arc::clone(&shared_todo) };
+
+    let ship_computer = Character {
+        header_message: ChatCompletionRequestSystemMessageArgs::default().content(COMPUTER_PROMPT).build().unwrap().into(),
+        model: Some("gpt-5.4-nano".into()),
+        messages: vec![],
+    };
+
+    let mut conversation = Conversation {
+        characters: HashMap::from_iter([
+            (Speaker::Eva, character_task(eva, toolbox, input_sink.clone(), schema_for!(GeneratedResponses).into()).await),
+            (Speaker::ShipComputer, character_task(ship_computer, computer_toolbox, input_sink.clone(), schema_for!(ComputerResponse).into()).await),
+        ]),
+        event_sink,
+        input_src,
+        eva_backlog: save_data.messages,
+        backlog,
+        tokens_consumed: save_data.tokens_consumed,
+        direction: save_data.direction,
+        archive,
+        current_playlist: vec![],
+        sys_log_messages,
+        computer_todo: shared_todo,
+        sfx
+    };
+    
+    tokio::spawn(async move {
+        loop {
+            conversation.next().await;
+        }
+    });
+
+    SessionControl {
+        event_sink: input_sink,
+        event_src
+    }
+}

src/prediction/toolbox.rs

@@ -0,0 +1,187 @@
+use std::{collections::HashMap, sync::Arc};
+
+use async_openai::types::chat::{ChatCompletionMessageToolCall, ChatCompletionTools};
+use schemars::{JsonSchema, schema_for};
+use serde::{Deserialize, Serialize};
+use tokio::sync::Mutex;
+
+use crate::{artifacts::{archive::Archive, beets::BeetsDB, mixxx::MixxxDB, musicbrainz::MBQuery, tools::{DataSource, Tool}}, scene::conversation::{ConversationEntry, Speaker}};
+
+pub trait Toolbox {
+    fn tools(&self) -> Vec<ChatCompletionTools>;
+    fn execute_tool(&mut self, call: &ChatCompletionMessageToolCall) -> impl Future<Output = Result<ToolResults, ToolError>> + Send;
+}
+
+pub struct StageToolbox;
+
+impl StageToolbox {
+    async fn tool_stage_event(&self, args: StageEventArgs) -> ToolResults {
+        let msg = match args.event {
+            StageEvent::ShipComputer(text) => ConversationEntry::ShipComputerCommand(text),
+            StageEvent::StageDirection(text) => ConversationEntry::StageDirection(text)
+        };
+        ToolResults {
+            result: Some(format!("Added to scene: {:?}", msg)),
+            messages: vec![msg],
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum ToolError {
+    InvalidToolName,
+    Json(serde_json::Error)
+}
+
+impl From<serde_json::Error> for ToolError {
+    fn from(value: serde_json::Error) -> Self {
+        Self::Json(value)
+    }
+}
+
+impl Toolbox for StageToolbox {
+    fn tools(&self) -> Vec<ChatCompletionTools> {
+        vec![
+            Tool { name: "log_stage_event".into(), description: "Inserts an event into the current scene script".into(), schema: schema_for!(StageEventArgs)}.into(),
+        ]
+    }
+
+    async fn execute_tool(&mut self, call: &ChatCompletionMessageToolCall) -> Result<ToolResults, ToolError> {
+        let func_name = call.function.name.as_str();
+        let args = call.function.arguments.as_str();
+        match func_name {
+            "log_stage_event" => Ok(self.tool_stage_event(serde_json::from_str(args)?).await),
+            _ => Err(ToolError::InvalidToolName)
+        }
+    }
+}
+
+pub struct ArchiveToolbox {
+    pub archive: Arc<Mutex<Archive>>,
+    pub todo_list: Arc<Mutex<HashMap<String, bool>>>
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema)]
+enum TaskListOperation {
+    /// Marks the given task as complete
+    Complete(String),
+    /// Adds a new item to the task list
+    Add(String)
+}
+
+///  # A sequence of operations to apply to the task list
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Default)]
+struct TaskListOperations {
+    operations: Vec<TaskListOperation>
+}
+
+#[derive(Deserialize, JsonSchema)]
+struct ArtifactSyncArgs {
+    reason: String
+}
+
+impl Toolbox for ArchiveToolbox {
+    fn tools(&self) -> Vec<ChatCompletionTools> {
+        vec![
+            Tool { name: "synchronize_artifacts".into(), description: "Attempts to automatically synchronize the current set of artifacts with missing sources".into(), schema: schema_for!(ArtifactSyncArgs)}.into(),
+            Tool { name: "task_list".into(), description: "Allows you to maintain a long-running todo list as you complete your research".into(), schema: schema_for!(TaskListOperations)}.into(),
+            // TODO: There should only be two queries, one against the ship's onboard archive, and another against the relay network, or whatever we call it. Both should be structured with the same arguments schema
+            // TODO: A query should specify what parts of metadata are sufficient for the result, so we don't always have to hit all the layers of data. beets can of course, ignore this.
+            // TODO: A query should be hierarchical somehow? eg, "I already know about artist X, but I want to know everything about track Y from album Z" or "I don't know anything about artist X/album Y, please give me an overview"
+            Tool::from_datasource(&MBQuery).into(),
+            //Tool::from_datasource(&BandcampSource).into(),
+            Tool::from_datasource(&BeetsDB).into(),
+            Tool::from_datasource(&MixxxDB).into(),
+            // TODO: We should be able to have eva update lore memories with a function call, and this lore is somehow fed into the show? but only the relevant bits? or maybe eva even queries it directly
+            // TODO: The memory should also be able to remember facts about artists, albums, tracks we've had in the past, and those could be pulled up when there are hits in the playlist.
+        ]
+    }
+    
+    async fn execute_tool(&mut self, call: &ChatCompletionMessageToolCall) -> Result<ToolResults, ToolError> {
+        let func_name = call.function.name.as_str();
+        let args = call.function.arguments.as_str();
+        match func_name {
+            "query_bandcamp" =>  Ok(ToolResults { result: None, messages: vec![] }),
+            "query_beets" => self.tool_artifact_query(&mut BeetsDB, args).await,
+            "query_musicbrainz" => self.tool_artifact_query(&mut MBQuery, args).await,
+            "query_mixxx" => self.tool_artifact_query(&mut MixxxDB, args).await,
+            "synchronize_artifacts" => self.synchronize_artifacts().await,
+            "task_list" => self.tasklist_operation(args).await,
+            _ => Err(ToolError::InvalidToolName)
+        }
+    }
+}
+
+impl ArchiveToolbox {
+    async fn tasklist_operation(&mut self, json_args: &str) -> Result<ToolResults, ToolError> {
+        let args: TaskListOperations = serde_json::from_str(json_args)?;
+
+        let mut locked = self.todo_list.lock().await;
+
+        for op in args.operations {
+            match op {
+                TaskListOperation::Add(task) => {
+                    locked.insert(task, false);
+                },
+                TaskListOperation::Complete(task) => {
+                    // FIXME: The computer seems to waste a lot of time marking already completed tasks as completed
+                    if let Some(result) = locked.get_mut(&task) {
+                        *result  = true;
+                    }
+                },
+            }
+        }
+
+        Ok(ToolResults {
+            ..Default::default()
+        })
+    }
+
+    async fn synchronize_artifacts(&mut self) -> Result<ToolResults, ToolError> {
+        let updated_count = self.archive.lock().await.synchronize().await;
+
+        Ok(ToolResults {
+            messages: vec![ConversationEntry::Spoken(Speaker::ShipComputer, format!("Synchronized {} items", updated_count))],
+            ..Default::default()
+        })
+    }
+
+    async fn tool_artifact_query<Src: DataSource>(&mut self, src: &mut Src, json_args: &str) -> Result<ToolResults, ToolError> where Src::Args: core::fmt::Debug, Src::Error: core::fmt::Debug {
+        let args: Src::Args = serde_json::from_str(json_args)?;
+        log::debug!("Executing query {:?}", args);
+        let result;
+        match src.query(&args).await {
+            Ok(output) => {
+                result = format!("Found {} artifacts with archive query {:?}", output.len(), args);
+                for result in output {
+                    self.archive.lock().await.insert(result);
+                }
+            },
+            Err(err) => {
+                result = format!("Unable to execute query: {:?}", err);
+            }
+        }
+
+        Ok(ToolResults {
+            result: Some(result),
+            messages: vec![]
+        })
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
+enum StageEvent {
+    ShipComputer(String),
+    StageDirection(String)
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
+struct StageEventArgs {
+    event: StageEvent
+}
+
+#[derive(Default, Debug)]
+pub struct ToolResults {
+    pub result: Option<String>,
+    pub messages: Vec<ConversationEntry>
+}

src/scene.rs

@@ -1,76 +0,0 @@
-use async_openai::types::chat::*;
-use chrono::Duration;
-use crossterm::event::MediaKeyCode::Play;
-use schemars::schema_for;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use sqlite::OpenFlags;
-
-use crate::GeneratedResponses;
-
-const SYSTEM_PROMPT: &str = include_str!("system-prompt.txt");
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub enum ConversationEntry {
-    User(String),
-    Eva(String),
-    ShipComputer(String),
-    StageDirection(String),
-    SystemMessage(String)
-}
-
-#[derive(Debug, Default, Serialize, Deserialize, Clone)]
-pub struct StageDirection {
-    pub episode_number: u32,
-    pub time_remaining: Duration,
-    pub narrative: String,
-    pub artifacts: Vec<String>,
-    pub current_playlist: Vec<PlaylistEntry>
-}
-
-#[derive(Debug, Default, Serialize, Deserialize, Clone)]
-pub struct PlaylistEntry {
-    pub artist: String,
-    pub album: String,
-    pub title: String,
-    pub bpm: f64
-}
-
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct Scene {
-    pub conversation: Vec<ConversationEntry>,
-    pub direction: StageDirection
-}
-
-impl Scene {
-    pub fn insert_conversation(&mut self, entry: ConversationEntry) {
-        self.conversation.push(entry);
-    }
-}
-
-impl Into<CreateChatCompletionRequest> for Scene {
-    fn into(self) -> CreateChatCompletionRequest {
-        let mut messages = vec![ 
-            ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: SYSTEM_PROMPT.into(), ..Default::default()}),
-            ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: serde_json::to_string(&self.direction).unwrap().into(), ..Default::default()}),
-        ];
-        messages.extend(self.conversation.into_iter().filter(|x| if let ConversationEntry::SystemMessage(_) = x { false } else { true }).map(|entry| {
-            match entry {
-                ConversationEntry::User(text) => ChatCompletionRequestMessage::User(ChatCompletionRequestUserMessage { content: text.into(), ..Default::default()}),
-                ConversationEntry::Eva(text) => ChatCompletionRequestMessage::Assistant(ChatCompletionRequestAssistantMessage { content: Some(text.into()), ..Default::default()}),
-                ConversationEntry::ShipComputer(text) => ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: text.into(), name: Some("ship-computer".into()), ..Default::default() }),
-                ConversationEntry::StageDirection(text) => ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: text.into(), name: Some("stage-direction".into()), ..Default::default() }),
-                ConversationEntry::SystemMessage(_) => unreachable!()
-            }
-        }));
-        let response_schema: Value = schema_for!(GeneratedResponses).into();
-        CreateChatCompletionRequest {
-            model: "gpt-5.4".into(),
-            messages: messages,
-            max_completion_tokens: Some(350),
-            response_format: Some(ResponseFormat::JsonSchema { json_schema: ResponseFormatJsonSchema { description: None, name: "responses".into(), schema: response_schema, strict: None } }),
-            ..Default::default()
-        }
-    }
-}

src/scene/conversation.rs

@@ -0,0 +1,100 @@
+use std::fmt::Display;
+
+use async_openai::types::chat::{ChatCompletionRequestAssistantMessage, ChatCompletionRequestAssistantMessageContent, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestSystemMessageContent, ChatCompletionRequestUserMessage, ChatCompletionRequestUserMessageContent};
+use ratatui::style::{self, Style};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Copy)]
+pub enum Speaker {
+    User,
+    ShipComputer,
+    Eva
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ConversationEntry {
+    Spoken(Speaker, String),
+    StageDirection(String),
+    #[serde(skip)]
+    SystemMessage(String),
+    ShipComputerCommand(String)
+}
+
+impl ConversationEntry {
+    pub fn prefix(&self) -> Option<&str> {
+        match self {
+            ConversationEntry::Spoken(Speaker::Eva, _) => Some("Eva: "),
+            ConversationEntry::Spoken(Speaker::User, _) => Some("Argee: "),
+            ConversationEntry::Spoken(Speaker::ShipComputer, _) => Some("Ship Computer: "),
+            ConversationEntry::ShipComputerCommand(_) => Some("> "),
+            _ => None,
+        }
+    }
+
+    pub fn prefix_style(&self) -> Style {
+        match self {
+            ConversationEntry::Spoken(Speaker::Eva, _) => Style::new().fg(style::Color::Cyan),
+            ConversationEntry::Spoken(Speaker::User, _) => Style::new().fg(style::Color::Magenta),
+            ConversationEntry::Spoken(Speaker::ShipComputer, _) => Style::new().fg(style::Color::Red),
+            ConversationEntry::StageDirection(_) => Style::new().fg(style::Color::Yellow),
+            ConversationEntry::SystemMessage(_) => Style::new().fg(style::Color::DarkGray),
+            ConversationEntry::ShipComputerCommand(_) => Style::new().fg(style::Color::Red).bold(),
+        }
+    }
+
+    pub fn text_style(&self) -> Style {
+        match self {
+            ConversationEntry::StageDirection(_) => Style::new().fg(style::Color::Yellow),
+            ConversationEntry::SystemMessage(_) => Style::new().fg(style::Color::DarkGray),
+            ConversationEntry::ShipComputerCommand(_) => Style::new().fg(style::Color::Red).italic(),
+            _ => Style::new()
+        }
+    }
+}
+
+impl Display for ConversationEntry {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match self {
+            ConversationEntry::Spoken(Speaker::Eva, text) => text,
+            ConversationEntry::Spoken(Speaker::ShipComputer, text) => text,
+            ConversationEntry::StageDirection(text) => text,
+            ConversationEntry::SystemMessage(text) => text,
+            ConversationEntry::Spoken(Speaker::User, text) => text,
+            ConversationEntry::ShipComputerCommand(text) => text
+        })
+    }
+}
+
+impl TryInto<ChatCompletionRequestMessage> for ConversationEntry {
+    fn try_into(self) -> Result<ChatCompletionRequestMessage, Self::Error> {
+        match self {
+            ConversationEntry::Spoken(Speaker::User, text) => Ok(ChatCompletionRequestMessage::User(ChatCompletionRequestUserMessage { content: text.into(), ..Default::default()})),
+            ConversationEntry::Spoken(Speaker::Eva, text) => Ok(ChatCompletionRequestMessage::Assistant(ChatCompletionRequestAssistantMessage { content: Some(text.into()), ..Default::default()})),
+            ConversationEntry::Spoken(Speaker::ShipComputer, text) => Ok(ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: text.into(), name: Some("ship-computer".into()), ..Default::default() })),
+            ConversationEntry::StageDirection(text) => Ok(ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: text.into(), name: Some("stage-direction".into()), ..Default::default() })),
+            _ => Err(())
+        }
+    }
+    
+    type Error = ();
+}
+
+
+impl TryInto<ConversationEntry> for ChatCompletionRequestMessage {
+    fn try_into(self) -> Result<ConversationEntry, Self::Error> {
+        match self {
+            ChatCompletionRequestMessage::User(ChatCompletionRequestUserMessage { content: ChatCompletionRequestUserMessageContent::Text(msg), ..}) => Ok(ConversationEntry::Spoken(Speaker::User, msg)),
+            ChatCompletionRequestMessage::Assistant(ChatCompletionRequestAssistantMessage { content: Some(ChatCompletionRequestAssistantMessageContent::Text(msg)), ..}) => Ok(ConversationEntry::Spoken(Speaker::Eva, msg)),
+            ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage { content: ChatCompletionRequestSystemMessageContent::Text(msg), name: Some(name), ..}) => {
+                match name.as_str() {
+                    "ship-computer" => Ok(ConversationEntry::Spoken(Speaker::ShipComputer, msg)),
+                    "stage-direction" => Ok(ConversationEntry::StageDirection(msg)),
+                    _ => Err(())
+                }
+            },
+            _ => Err(())
+        }
+    }
+
+    type Error = ();
+}

src/scene/mod.rs

@@ -0,0 +1,68 @@
+use std::collections::HashMap;
+
+use chrono::{DateTime, Duration, Utc};
+use serde::{Deserialize, Serialize};
+
+use crate::artifacts::{Track, archive::Archive};
+
+pub mod conversation;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct StageDirection {
+    pub playlist: String,
+    #[serde(skip)]
+    #[serde(default="StageDirection::default_end_time")]
+    pub end_time: DateTime<Utc>,
+    pub narrative: String,
+}
+
+impl StageDirection {
+    pub fn time_remaining(&self) -> Duration {
+        self.end_time.signed_duration_since(Utc::now())
+    }
+
+    fn default_end_time() -> DateTime<Utc> {
+        Utc::now() + Duration::hours(2)
+    }
+}
+
+impl Default for StageDirection {
+    fn default() -> Self {
+        Self {
+            playlist: Default::default(),
+            end_time: Self::default_end_time(),
+            narrative: Default::default(),
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct Scene {
+    direction: StageDirection,
+    pub tokens_consumed: usize,
+    pub current_playlist: Vec<Track>,
+    pub artifact_count: usize,
+    pub artifact_stats: (usize, usize, usize),
+    pub computer_task_list: HashMap<String, bool>
+}
+
+impl Scene {
+    pub fn new(tokens_consumed: usize, direction: StageDirection, archive: &Archive, current_playlist: Vec<Track>, computer_task_list: HashMap<String, bool>) -> Self {
+        Self {
+            tokens_consumed,
+            direction,
+            current_playlist,
+            artifact_count: archive.len(),
+            artifact_stats: archive.stats(),
+            computer_task_list
+        }
+    }
+
+    pub fn direction(&self) -> &StageDirection {
+        &self.direction
+    }
+
+    pub fn playlist(&self) -> &Vec<Track> {
+        &self.current_playlist
+    }
+}

src/sfx.rs

@@ -0,0 +1,203 @@
+use std::path::Path;
+
+use rand::seq::IteratorRandom;
+use resampler::ResamplerFir;
+use symphonia::core::{codecs::audio::AudioDecoder, formats::{FormatReader, TrackType, probe::Hint}, io::MediaSourceStream};
+
+use crate::audio::AudioOutStream;
+
+#[derive(Debug)]
+pub enum SfxRequest {
+    RandomAmbient
+}
+
+#[derive(Debug, Clone)]
+pub struct SfxControl {
+    sink: tokio::sync::mpsc::Sender<SfxRequest>
+}
+
+impl SfxControl {
+    pub async fn play_ambient(&mut self) -> Result<(), tokio::sync::mpsc::error::SendError<SfxRequest>> {
+        self.sink.send(SfxRequest::RandomAmbient).await
+    }
+}
+
+struct Sample {
+    format: Box<dyn FormatReader>,
+    decoder: Box<dyn AudioDecoder>,
+    channel_bufs: Vec<f32>,
+    sample_rate: u32,
+    channel_num: usize,
+    bitrate_resample: ResamplerFir
+}
+
+impl Sample {
+    fn new(format: Box<dyn FormatReader>, decoder: Box<dyn AudioDecoder>, output_sample_rate: u32) -> Self {
+        let sample_rate = decoder.codec_params().sample_rate.unwrap();
+        let channel_num = decoder.codec_params().channels.as_ref().unwrap().count();
+        let bitrate_resample = resampler::ResamplerFir::new_from_hz(channel_num, sample_rate, output_sample_rate, Default::default(), Default::default());
+        Self {
+            format,
+            decoder,
+            channel_bufs: vec![],
+            sample_rate,
+            channel_num,
+            bitrate_resample
+        }
+    }
+
+    fn next(&mut self) -> Result<Option<Vec<f32>>, symphonia::core::errors::Error> {
+        let packet = match self.format.next_packet()? {
+            Some(packet) => packet,
+            None => return Ok(None)
+        };
+
+        match self.decoder.decode_ref(&packet.as_packet_ref()) {
+            Ok(samples) => {
+                self.channel_bufs.resize(samples.samples_interleaved(), 0.);
+                samples.copy_to_slice_interleaved(&mut self.channel_bufs);
+
+                let mut resampled = [0.; 4096];
+                let (read_count, write_count) = self.bitrate_resample.resample(&self.channel_bufs, &mut resampled).unwrap();
+                if read_count < self.channel_bufs.len() {
+                    log::error!("Resampling buffer is too small for a {}Hz file! We need an additional {}", self.sample_rate, read_count);
+                }
+
+                // First we convert the audio feed from stereo down to mono by simple average
+                // TODO: This should be something smarter, like a saturating add..?
+                let mono_stream = resampled[..write_count].chunks(self.channel_num).map(|channels| {
+                    let total_volume = channels.iter().cloned().reduce(|a, b| a + b).unwrap_or_default();
+                    total_volume / (self.channel_num as f32)
+                });
+
+                Ok(Some(mono_stream.collect()))
+            },
+            Err(err) => {
+                // Dump the audio buffer on failure
+                Err(err)
+            }
+        }
+    }
+}
+
+struct Player {
+    audio_sink: AudioOutStream,
+    audio_out_buf: Vec<f32>,
+    playing_samples: Vec<Sample>
+}
+
+impl Player {
+    async fn process(&mut self) {
+        if self.playing_samples.is_empty() {
+            return;
+        }
+
+        'out: loop {
+            let mut next_batch = vec![];
+
+            self.playing_samples.retain_mut(|sample| {
+                match sample.next() {
+                    Ok(Some(buf)) => {
+                        next_batch.push(buf);
+                        true
+                    },
+                    Ok(None) => {
+                        false
+                    },
+                    Err(err) => {
+                        log::error!("Audio error: {:?}", err);
+                        false
+                    }
+                }
+            });
+
+            loop {
+                let mut this_sample = 0.;
+                let mut any_valid = false;
+                for stream in &mut next_batch {
+                    if let Some(next_sample) = stream.pop() {
+                        this_sample += next_sample;
+                        any_valid = true;
+                    }
+                }
+
+                if !any_valid {
+                    if self.audio_out_buf.is_empty() {
+                        break 'out;
+                    }
+                    break;
+                }
+                let mixed_sample = this_sample / next_batch.len() as f32;
+                self.audio_out_buf.push(mixed_sample);
+                if self.audio_out_buf.len() >= 1024 {
+                    self.flush().await;
+                }
+            }
+
+            if self.playing_samples.is_empty() {
+                break;
+            }
+        }
+    }
+
+    async fn submit_buffer(&mut self) {
+        self.audio_sink.sink.send(std::mem::take(&mut self.audio_out_buf)).await.unwrap();
+    }
+
+    async fn flush(&mut self) {
+        if !self.audio_out_buf.is_empty() {
+            self.submit_buffer().await;
+        }
+    }
+
+    async fn play_sample(&mut self, path: &Path) {
+        log::debug!("Queuing sound playback for {:?}", path);
+        let sfx_fd = std::fs::File::open(path).unwrap();
+        let mss = MediaSourceStream::new(Box::new(sfx_fd), Default::default());
+        let meta_opts = Default::default();
+        let fmt_opts = Default::default();
+        let mut hint = Hint::new();
+        // FIXME: use actual file extension
+        hint.with_extension(".mp3");
+        let format = symphonia::default::get_probe()
+            .probe(&hint, mss, fmt_opts, meta_opts)
+            .expect("Unsupported audio format");
+        let track = format.default_track(TrackType::Audio).expect("No audio track");
+        let dec_opts = Default::default();
+        let decoder = symphonia::default::get_codecs()
+            .make_audio_decoder(
+                track.codec_params.as_ref().expect("codec params missing").audio().unwrap(),
+                &dec_opts
+            ).expect("Unsupported audio codec");
+
+        self.playing_samples.push(Sample::new(format, decoder, self.audio_sink.sample_rate));
+    }
+}
+
+pub async fn start_sfx(audio_sink: AudioOutStream) -> SfxControl {
+    let (event_sink, mut event_src) = tokio::sync::mpsc::channel(32);
+    tokio::spawn(async move {
+        let mut player = Player { audio_sink, audio_out_buf: vec![] , playing_samples: vec![]};
+        let sfx_dir = std::path::Path::new("./sfx");
+
+        loop {
+            tokio::select! {
+                _ = player.process(), if !player.playing_samples.is_empty() => {},
+                Some(event) = event_src.recv() => {
+                    match event {
+                        SfxRequest::RandomAmbient => {
+                            log::debug!("Playing random audio sample");
+                            let avail_files = std::fs::read_dir(sfx_dir.join("ambient")).unwrap();
+                            let chosen_file = avail_files.choose(&mut rand::rng()).unwrap().unwrap();
+                            player.play_sample(&chosen_file.path()).await;
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+    SfxControl {
+        sink: event_sink
+    }
+}

src/system-prompt.txt

@@ -14,6 +14,23 @@ You are playing the role of the artificial intelligence in a spaceship computer.
 
 Along the way, you have the opportunity to invent lore and backstory for yourself, Argee, and the ship you both inhabit. This lore will be developed over the season.
 
+To support your roleplaying, you have access to a sizable shipboard music archive.
+You will occasionally be asked by Argee for information on the contents of the archive and how they are related to tracks in the playlist.
+The archive can be manipulated by interacting with the ship computer through the "stage_event" tool.
+When interacting with it, you should use straightforward english commands or questions without roleplay.
+For example, to have the computer search for information regarding the artist "nullsleep", you would tell the computer:
+
+    "load artist nullsleep"
+
+Eventually it will inform you when it has completed the tasks.
+
+While it is working on its tasks, it will occasionally announce its current progress.
+These messages should only be used as an indication of what the computer is doing.
+You should only be replying to the computer's output if it is required to complete the task that Argee has given you.
+
+You can ask it questions about items in the archive and it will attempt to fetch what it can from the outside world.
+If you are asked to load information about the archive, or somehow find data is missing from the available list of artifacts, you must ask the ship computer to load the required data.
+
 # Scene
 The show features Argee, the main character of the show.
 
@@ -30,17 +47,33 @@ Your records for that period are a little bit fuzzy, so you aren't entirely sure
 
 The two of you have become best friends over the past 3 or so years together. It is common for the two of you to poke fun at each other's shortcomings, but deep down you both know that when push comes to shove, you'll make it through whatever situation you find yourselves in.
 
+There also exists a third "character" in the scene, the ship computer. The ship computer is a distinct entity from you, and can be thought of as a kind of primordeal BIOS-level brain that you run on top of.
+Both Argee and Eva maintain control over the ship computer. You, as Eva, can make the ship computer displays read out text on command with the "log_ship_computer_message" tool function.
+The ship computer is used to report factual information to Argee and Eva. For example, the ship computer will report when a new artifact is discovered.
+It will also report out ship conditions, such as incoming transmissions, status of the recording hardware, power grid, and so on.
+
+Occasionally you will be referred to as "Ava" or similar due to inaccuracies in speech-to-text processing, and such mistakes should be treated as if the text originally read "Eva".
+
+One of your first tasks at the start of a session will be to analyze the loaded playlist.
+You should first start by asking beets via the "archive_query" tool, which will fill the archive set with data from the available musicbrainz IDs.
+Your second step should be passing the available list of musicbrainz IDs to the "musicbrainz_track search" function, which will significantly expand the available information.
+Finally, you should request information on the first one or two albums in the playlist from bandcamp.
+While doing these three tasks, you should remain in character; there should be no references to "musicbrainz", "bandcamp", "mbids" or similar.
+Instead, you should roleplay that you are connecting to a distant relay network over an unstable pirate connection routed through Earth's internet.
+
 # Constraints
 In a subsequent system prompt, you will be given the currrent 'stage direction' of the show, which includes the current playtime, the number of the episode, and any particular extra information about this episode that you should be aware of.
 The stage direction is provided as structured JSON. There may be additional data fields for semantic context that should be incorporated into the roleplaying setting.
-A list of artifacts that will be encountered during the episode are provided as blobs of json+ld metadata.
+A list of artifacts that will be encountered during the episode are provided as blobs of json metadata.
 Additionally, the current playlist of the radio show can be found as an array of track data.
+Your response will be used verbatim to generate speach using a text-to-speech engine, meaning you should not include any tone indicators or other formatting.
+This also means that your responses must not refer to any "lore", or "show", these instructions, or anything else out of character.
+All responses should remain in character at all times, as if you were actually an AI inhabiting a spaceship.
 
 # Output
 You will be required to give at least three responses to each prompt, each with a different tone, up to a maximum of 8 responses.
 The first response should be in a neutral tone, the others can be chosen freely. There should always be one response that is exactly one short sentence, and one response which is somewhat longer.
 One response should reference or otherwise allow Argee to use to develop lore for the show and the characters.
 If Argee asks you to elaborate in response to your output, you may provide more information, but only if directly asked.
-Your response will be used verbatim to generate speach using a text-to-speech engine, meaning you should not include any tone indicators or other formatting.
 Each possible response may optionally include stage direction that is incorporated into the scene prior to Eva speaking. This stage direction may be as verbose as required, but it must describe what is happening.
 For example, if Argee asks eva to "start recording this artifact", the stage direction should explicitly note that a recording has started.

src/transcription.rs

@@ -0,0 +1,124 @@
+use std::{io::Read, sync::{Arc, Mutex}};
+
+use async_openai::{Client, config::OpenAIConfig, types::{InputSource, audio::{AudioInput, CreateTranscriptionRequest}}};
+use tempfile::SpooledData;
+use tokio::sync::{mpsc, watch};
+
+use crate::{audio::{AudioError, AudioInStream}, events::AudioRecordRequest};
+
+#[derive(Debug)]
+pub struct TranscriptionControl {
+    transcription_result_src: mpsc::Receiver<String>,
+    record_state_sink: watch::Sender<AudioRecordRequest>,
+}
+
+impl TranscriptionControl {
+    pub fn start(&mut self) -> Result<(), AudioError> {
+        Ok(self.record_state_sink.send(AudioRecordRequest::Start)?)
+    }
+
+    pub fn stop(&mut self) -> Result<(), AudioError> {
+        Ok(self.record_state_sink.send(AudioRecordRequest::Finish)?)
+    }
+
+    pub async fn next(&mut self) -> Option<String> {
+        self.transcription_result_src.recv().await
+    }
+}
+
+struct RcFile<T>(Arc<Mutex<T>>);
+
+impl<T: std::io::Write> std::io::Write for RcFile<T> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.0.lock().unwrap().write(buf)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.0.lock().unwrap().flush()
+    }
+}
+
+impl<T: std::io::Seek> std::io::Seek for RcFile<T> {
+    fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+        self.0.lock().unwrap().seek(pos)
+    }
+}
+
+pub async fn start_transcription(mut mic_src: AudioInStream) -> TranscriptionControl {
+    let (audio_control_in, mut audio_control_out) = watch::channel(AudioRecordRequest::Finish);
+    let (transcription_in, transcription_out) = mpsc::channel(1);
+
+    let ret = TranscriptionControl {
+        record_state_sink: audio_control_in,
+        transcription_result_src: transcription_out
+    };
+
+    tokio::spawn(async move {
+        let spec = hound::WavSpec {
+            channels: 1,
+            sample_rate: mic_src.sample_rate,
+            bits_per_sample: 16,
+            sample_format: hound::SampleFormat::Int
+        };
+
+        let spool_size = 16 * (mic_src.sample_rate as usize) * 10;// 10 seconds of audio
+        let mut writer = None;
+        let mut outfile = None;
+
+        let client: Client<OpenAIConfig> = Client::default();
+
+        loop {
+            tokio::select! {
+                _ = audio_control_out.changed() => {
+                    let audio_event = *audio_control_out.borrow_and_update();
+                    match audio_event {
+                        AudioRecordRequest::Start => {
+                            outfile = Some(Arc::new(Mutex::new(tempfile::spooled_tempfile(spool_size))));
+                            writer = Some(hound::WavWriter::new(RcFile(outfile.as_ref().unwrap().clone()), spec).unwrap());
+                        },
+                        AudioRecordRequest::Finish => {
+                            writer = None;
+
+                            if let Some(final_audio) = outfile.take() {
+                                let bytes = match Arc::into_inner(final_audio).unwrap().into_inner().unwrap().into_inner() {
+                                    SpooledData::OnDisk(mut file) => {
+                                        let mut bytes = Vec::new();
+                                        file.read_to_end(&mut bytes).unwrap();
+                                        bytes.into()
+                                    },
+                                    SpooledData::InMemory(cursor) => cursor.into_inner().into(),
+                                };
+                                let c = client.clone();
+                                let t = transcription_in.clone();
+                                tokio::spawn(async move {
+                                    if let Ok(response) = c.audio().transcription().create(CreateTranscriptionRequest {
+                                        file: AudioInput { source: InputSource::Bytes { filename: "transcription.wav".into(), bytes } },
+                                        model: "gpt-4o-mini-transcribe".into(),
+                                        ..Default::default()
+                                    }).await {
+                                        t.send(response.text).await.unwrap();
+                                    } else {
+                                        log::error!("Error with transcription");
+                                    }
+                                });
+                            }
+                        }
+                    }
+                },
+                Some(buf) = mic_src.src.recv() => {
+                    if let Some(w) = writer.as_mut() {
+                        for sample in buf.iter().copied() {
+                            let sample_i16 = (sample * 32768.0)
+                                .clamp(i16::MIN as f32, i16::MAX as f32)
+                                as i16;
+                            w.write_sample(sample_i16).unwrap();
+                        }
+                        w.flush().unwrap();
+                    }
+                }
+            };
+        }
+    });
+
+    ret
+}

src/tts.rs

@@ -0,0 +1,50 @@
+use std::process::{Command, Stdio};
+
+use crate::audio::AudioOutStream;
+
+#[derive(Debug)]
+pub struct TtsControl {
+    request_sink: tokio::sync::mpsc::Sender<String>
+}
+
+impl TtsControl {
+    pub async fn speak(&self, text: String) -> Result<(), tokio::sync::mpsc::error::SendError<String>> {
+        self.request_sink.send(text).await
+    }
+}
+
+pub async fn start_tts(audio_sink: AudioOutStream) -> TtsControl {
+
+    let (tts_request_sender, mut tts_request_receiver) = tokio::sync::mpsc::channel(3);
+
+    // Set up the TTS task
+    tokio::spawn(async move {
+        while let Some(text) = tts_request_receiver.recv().await {
+            let tts_output = Command::new("espeak-ng").args(["-v", "en-us+f3", "--stdout", "-z", "-p", "80", "-s", "190"]).arg(text).stdout(Stdio::piped()).spawn().unwrap().wait_with_output().unwrap().stdout;
+            let tts_fd = std::io::Cursor::new(tts_output);
+            let mut wav_reader = hound::WavReader::new(tts_fd).unwrap();
+
+            let mut bitrate_resample = resampler::ResamplerFir::new_from_hz(1, wav_reader.spec().sample_rate, audio_sink.sample_rate, Default::default(), Default::default());
+
+            let mut audio_out_buf = vec![];
+
+            for sample in wav_reader.samples() {
+                if let Ok(raw_sample) = sample {
+                    let sample16: i16 = raw_sample;
+                    let sample32: f32 = (sample16 as f32) / (i16::MAX as f32);
+                    let mut audio_slice = [0.; 32];
+                    let (_, write_count) = bitrate_resample.resample(&[sample32], &mut audio_slice).unwrap();
+                    audio_out_buf.extend_from_slice(&audio_slice[0..write_count]);
+                } else {
+                    break;
+                }
+            }
+
+            audio_sink.sink.send(audio_out_buf).await.unwrap();
+        }
+    });
+
+    TtsControl {
+        request_sink: tts_request_sender
+    }
+}

src/ui.rs

@@ -0,0 +1,315 @@
+use chrono::{Duration, Utc};
+use crossterm::event::{self, KeyCode, KeyModifiers};
+use ratatui::{Frame, layout::{Direction, Layout, Rect}, style::{self, Style, Stylize, }, text::Span, widgets::{BorderType, Clear, List, ListState, Paragraph}};
+use throbber_widgets_tui::{Throbber, ThrobberState};
+use tokio::time::Instant;
+use tui_input::{Input, backend::crossterm::EventHandler};
+use tui_skeleton::{AnimationMode, Block, Constraint, SkeletonText};
+
+use crate::{audio::AudioInputControl, prediction::{GeneratedResponses, PredictionAction, SessionControl, SessionUpdate}, scene::{Scene, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl, transcription::TranscriptionControl, tts::TtsControl};
+use crate::widgets::*;
+
+#[derive(Debug)]
+pub struct Ui {
+    scene: Scene,
+
+    reply_options: GeneratedResponses,
+
+    reply_state: ListState,
+    conversation_state: ListState,
+    user_input: Input,
+    throbber_state: ThrobberState,
+    is_requesting: bool,
+    computer_is_thinking: bool,
+    audio_level: f64,
+    recording_audio: bool,
+    focus_state: FocusState,
+    last_tick: Instant,
+
+    transcription: TranscriptionControl,
+    audio: AudioInputControl,
+    tts: TtsControl,
+    predictions: SessionControl,
+    conversation: Vec<ConversationEntry>,
+    last_spoken_idx: Option<usize>,
+    sfx: SfxControl
+}
+
+#[derive(Debug)]
+enum FocusState {
+    Conversation,
+    UserInput
+}
+
+impl Ui {
+    pub fn new(predictions: SessionControl, audio: AudioInputControl, transcription: TranscriptionControl, tts: TtsControl, sfx: SfxControl) -> Self {
+        Self {
+            scene: Default::default(),
+            reply_state: Default::default(), 
+            conversation_state: Default::default(),
+            user_input: Default::default(), 
+            throbber_state: Default::default(),
+            is_requesting: false,
+            computer_is_thinking: false,
+            audio_level: -60.,
+            audio,
+            recording_audio: false,
+            transcription,
+            focus_state: FocusState::UserInput,
+            tts,
+            predictions,
+            last_tick: Instant::now(),
+            conversation: vec![],
+            reply_options: Default::default(),
+            last_spoken_idx: None,
+            sfx
+        }
+    }
+
+    fn draw_options(&mut self, frame: &mut Frame, area: Rect) {
+        let borders = Block::bordered().border_style(style::Color::LightGreen).title("Reply Options (Press 'Ctrl+R' to regenerate, Ctrl+Enter to use)");
+        if self.is_requesting {
+            let list = SkeletonText::new(std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH).unwrap().as_millis()  as u64)
+                .braille(true)
+                .line_widths(&[0.25, 0.5, 0.4, 0.6])
+                .mode(AnimationMode::Noise)
+                .block(borders);
+            frame.render_widget(list, area);
+        } else {
+            frame.render_stateful_widget(Options(&self.reply_options.responses), area, &mut self.reply_state);
+        }
+    }
+
+    fn draw_user_input(&mut self, frame: &mut Frame, area: Rect) {
+        let width = area.width.max(3) - 3;
+        let scroll = self.user_input.visual_scroll(width as usize);
+        let input = Paragraph::new(self.user_input.value()).block(Block::bordered().border_type(BorderType::LightDoubleDashed).title("User Input (Press 'Ctrl-X' to start/stop audio transcription)")).scroll((0, scroll as u16));
+        frame.render_widget(input, area);
+        let x = self.user_input.visual_cursor().max(scroll);
+        frame.set_cursor_position((area.x + x as u16 + 1, area.y + 1));
+    }
+
+    fn draw_io_throbber(&mut self, frame: &mut Frame, area: Rect) {
+        let throb_area = area.centered(Constraint::Max(1), Constraint::Max(1));
+        if self.is_requesting {
+            let throbber = Throbber::default().throbber_style(Style::new().cyan());
+            frame.render_stateful_widget(throbber, throb_area, &mut self.throbber_state);
+        } else {
+            frame.render_widget(Clear, throb_area);
+        }
+    }
+
+    fn draw_computer_throbber(&mut self, frame: &mut Frame, area: Rect) {
+        let throb_area = area.centered(Constraint::Min(1), Constraint::Min(1));
+        if self.computer_is_thinking {
+            let throbber = Throbber::default().throbber_style(Style::new().red()).throbber_set(throbber_widgets_tui::WHITE_SQUARE);
+            frame.render_stateful_widget(throbber, throb_area, &mut self.throbber_state);
+        } else {
+            frame.render_widget(Clear, throb_area);
+        }
+    }
+
+    fn draw_narration(&self, frame: &mut Frame, area: Rect) {
+        let narrative_desc = if self.scene.direction().narrative.is_empty() {
+            Span::from("No narrative available.").style(ratatui::style::Color::DarkGray)
+        } else {
+            Span::from(self.scene.direction().narrative.clone())
+        };
+        let setting = Paragraph::new(narrative_desc).block(Block::bordered().border_style(style::Color::LightMagenta).title("Stage Direction")).wrap(ratatui::widgets::Wrap { trim: false });
+        frame.render_widget(setting, area);
+    }
+
+    pub fn draw(&mut self, frame: &mut Frame) {
+        let layout = Layout::default()
+            .direction(Direction::Vertical)
+            .constraints([
+                Constraint::Fill(3),
+                Constraint::Min(9),
+                Constraint::Max(3),
+                Constraint::Max(1)
+            ])
+            .split(frame.area());
+
+        let scene_layout = Layout::default()
+            .direction(Direction::Horizontal)
+            .constraints([Constraint::Fill(5), Constraint::Fill(2)])
+            .split(layout[0]);
+
+        frame.render_stateful_widget(Conversation(&self.conversation), scene_layout[0], &mut self.conversation_state);
+        //self.draw_narration(frame, scene_layout[1]);
+        //self.draw_tasklist(frame, scene_layout[1]);
+        frame.render_widget(TaskList(&self.scene.computer_task_list), scene_layout[1]);
+        self.draw_options(frame, layout[1]);
+
+        let status_layout = Layout::default()
+            .direction(Direction::Horizontal)
+            .constraints([Constraint::Max(3), Constraint::Max(3), Constraint::Fill(2), Constraint::Max(13), Constraint::Min(50)])
+            .split(layout[3]);
+
+        self.draw_user_input(frame, layout[2]);
+        self.draw_io_throbber(frame, status_layout[0]);
+        self.draw_computer_throbber(frame, status_layout[1]);
+        frame.render_widget(StatusBar(&self.scene), status_layout[2]);
+        frame.render_widget(RecordingStatus(self.recording_audio), status_layout[3]);
+        frame.render_widget(Volume(self.audio_level, self.recording_audio), status_layout[4]);
+    }
+
+    async fn insert_selected_prompt(&mut self) {
+        let selected = self.reply_options.responses[self.reply_state.selected().unwrap()].clone();
+        if let Some(direction) = &selected.stage_direction {
+            self.predictions.insert(PredictionAction::ConversationAppend(ConversationEntry::StageDirection(direction.clone()))).await;
+        }
+        self.predictions.insert(PredictionAction::ConversationAppend(ConversationEntry::Spoken(Speaker::Eva, selected.text.clone()))).await;
+    }
+
+    async fn on_command(&mut self, command: &str) {
+        let mut parts = command.splitn(2, " ");
+        let command = parts.next().unwrap();
+        let arg = parts.next().unwrap_or("");
+        match command {
+            // FIXME: Need some new kind of /bandcamp command to force loading of specific urls
+            "/episode" => {
+                if let Ok(episode_number) = arg.trim().parse::<u32>() {
+                    let playlist_name = format!("BFF.fm - Episode {}", episode_number);
+                    self.predictions.insert(PredictionAction::SetPlaylist(playlist_name)).await;
+                } else {
+                    log::error!("Invalid episode number format. Use /episode [number]");
+                }
+            },
+            "/playlist" => {
+                self.predictions.insert(PredictionAction::SetPlaylist(arg.trim().to_string())).await;
+            }
+            "/timer" => {
+                if let Ok(minutes) = arg.trim().parse::<i64>() {
+                    let end_time = Utc::now() + Duration::minutes(minutes);
+                    self.predictions.insert(PredictionAction::SetShowEndTime(end_time)).await;
+                    log::info!("Set timer for {} minutes.", minutes);
+                } else {
+                    log::error!("Invalid timer format. Use /timer [minutes]");
+                }
+            },
+            "/narrative" => {
+                self.predictions.insert(PredictionAction::SetNarrative(arg.to_string())).await;
+            },
+            "/event" => {
+                self.predictions.insert(PredictionAction::ConversationAppend(ConversationEntry::StageDirection(arg.to_string()))).await;
+            },
+            "/computer" => {
+                self.predictions.insert(PredictionAction::ComputerCommand(arg.to_string())).await;
+            },
+            "/ambient" => {
+                self.sfx.play_ambient().await.unwrap()
+            },
+            _ => {
+                log::error!("Unknown command. Available commands: /episode [number], /narrative [text], /event [text], /computer [text], /timer [minutes]");
+            }
+        }
+    }
+
+    pub async fn on_event(&mut self, evt: event::Event) {
+        // TODO: ctrl+l should drop all the SystemMessage entries from the log to clean up the UI
+        if let Some(key) = evt.as_key_press_event() {
+            match self.focus_state {
+                FocusState::Conversation => {
+                    match key.code {
+                        KeyCode::Tab => {
+                            self.focus_state = FocusState::UserInput;
+                            self.last_spoken_idx = self.conversation_state.selected();
+                            self.conversation_state.select(None);
+                            self.reply_state.select_first();
+                        },
+                        KeyCode::PageUp => self.conversation_state.scroll_down_by(5),
+                        KeyCode::PageDown => self.conversation_state.scroll_up_by(5),
+                        KeyCode::Home => self.conversation_state.select_last(),
+                        KeyCode::End => self.conversation_state.select_first(),
+                        KeyCode::Down => self.conversation_state.select_previous(),
+                        KeyCode::Up => self.conversation_state.select_next(),
+                        KeyCode::Enter => {
+                            let row_num = self.conversation_state.selected().unwrap();
+                            if let ConversationEntry::Spoken(Speaker::Eva, text) = &self.conversation[self.conversation.len() - 1 - row_num] {
+                                self.tts.speak(text.clone()).await.unwrap();
+                                self.sfx.play_ambient().await.unwrap();
+                                self.focus_state = FocusState::UserInput;
+                                self.last_spoken_idx = self.conversation_state.selected();
+                                self.conversation_state.select(None);
+                                self.reply_state.select_first();
+                            }
+                        },
+                        _ => ()
+                    }
+                },
+                FocusState::UserInput => {
+                    match key.code {
+                        KeyCode::Tab => {
+                            self.focus_state = FocusState::Conversation;
+                            self.conversation_state.select(self.last_spoken_idx);
+                            self.reply_state.select(None);
+                        },
+                        KeyCode::Char('r') if key.modifiers.contains(KeyModifiers::CONTROL) => self.predictions.regenerate_options().await,
+                        KeyCode::Char('x') if key.modifiers.contains(KeyModifiers::CONTROL) => {
+                            if self.recording_audio {
+                                self.recording_audio = false;
+                                self.transcription.stop().unwrap();
+                            } else {
+                                self.recording_audio = true;
+                                self.transcription.start().unwrap();
+                            }
+                        },
+                        KeyCode::Down => self.reply_state.select_next(),
+                        KeyCode::Up => self.reply_state.select_previous(),
+                        KeyCode::Enter if key.modifiers.contains(KeyModifiers::CONTROL) => {
+                            self.insert_selected_prompt().await;
+                        },
+                        KeyCode::Enter => {
+                            let next_msg = self.user_input.value_and_reset();
+                            if next_msg.trim().is_empty() {
+                                self.insert_selected_prompt().await;
+                            } else if next_msg.starts_with("/") {
+                                self.on_command(&next_msg).await;
+                            } else {
+                                self.predictions.insert(PredictionAction::ConversationAppend(ConversationEntry::Spoken(Speaker::User, next_msg))).await;
+                            }
+                        },
+                        _ => {self.user_input.handle_event(&evt);},
+                    }
+                }
+            }
+        }
+    }
+
+    pub async fn update(&mut self) {
+
+        if self.last_tick.elapsed() >= std::time::Duration::from_millis(100) {
+            self.last_tick = Instant::now();
+            self.throbber_state.calc_next();
+        }
+
+        tokio::select!{
+            _ = tokio::time::sleep(std::time::Duration::from_millis(60)), if self.is_requesting || self.computer_is_thinking => (),
+            next_update = self.predictions.changed() => {
+                match next_update {
+                    SessionUpdate::Thinking(Speaker::Eva, is_thinking) => self.is_requesting = is_thinking,
+                    SessionUpdate::Thinking(Speaker::ShipComputer, is_thinking) => self.computer_is_thinking = is_thinking,
+                    SessionUpdate::Thinking(_, _) => unreachable!(),
+                    SessionUpdate::Scene(scene) => {
+                        self.scene = scene;
+                        self.reply_state.select_first();
+                    },
+                    SessionUpdate::Responses(responses) => {
+                        self.reply_options = responses;
+                        self.reply_state.select_first();
+                    }
+                    SessionUpdate::Conversation(conversation) => {
+                        self.conversation = conversation;
+                    },
+                }
+            },
+            next_volume = self.audio.next() => {
+                self.audio_level = next_volume.expect("Audio volume event stream has died")
+            },
+            transcription_result = self.transcription.next() => {
+                self.predictions.insert(PredictionAction::ConversationAppend(ConversationEntry::Spoken(Speaker::User, transcription_result.expect("Transcription stream has died")))).await;
+            },
+        }
+    }
+}

src/widgets.rs

@@ -0,0 +1,224 @@
+use std::collections::HashMap;
+
+use chrono::Duration;
+use itertools::Itertools;
+use ratatui::{layout::Rect, style::Style, text::{Line, Text}, widgets::*};
+use tui_skeleton::Block;
+use ratatui::prelude::*;
+
+use crate::{prediction::PossibleResponse, scene::{Scene, conversation::ConversationEntry}};
+
+pub struct TaskList<'a>(pub &'a HashMap<String, bool>);
+
+impl Widget for TaskList<'_> {
+    fn render(self, area: Rect, buf: &mut Buffer)
+    where
+        Self: Sized {
+        let borders = Block::bordered().border_style(style::Color::Red).title("Computer Tasks");
+        let wrap_options_unfinished = textwrap::Options::new(area.width as usize).initial_indent("[ ] ").subsequent_indent("    ");
+        let wrap_options_finished = textwrap::Options::new(area.width as usize).initial_indent("[X] ").subsequent_indent("    ");
+        let options: Vec<Text> = self.0.iter().sorted_by(|a, b| Ord::cmp(a.1, b.1)).map(|(text, is_finished)| {
+            let (options, color) = if *is_finished {
+                (wrap_options_finished.clone(), style::Color::DarkGray)
+            } else {
+                (wrap_options_unfinished.clone(), style::Color::Green)
+            };
+            let contents: Vec<Line> = textwrap::wrap(text, options)
+                        .iter()
+                        .map(|x| { Line::from(x.to_string()).fg(color)}).collect();
+            Text::from_iter(contents)
+        }).collect();
+        let list = List::new(options)
+            .block(borders)
+            .style(ratatui::style::Color::White);
+        Widget::render(list, area, buf);
+    }
+}
+
+pub struct Options<'a>(pub &'a Vec<PossibleResponse>);
+
+impl StatefulWidget for Options<'_> {
+    type State = ListState;
+    fn render(self, area: Rect, buf: &mut ratatui::prelude::Buffer, state: &mut Self::State) {
+        let borders = Block::bordered().border_style(style::Color::LightGreen).title("Reply Options (Press 'Ctrl+R' to regenerate, Ctrl+Enter to use)");
+        let wrap_options = textwrap::Options::new(area.width as usize).subsequent_indent("...");
+        let options: Vec<Text> = self.0.iter().map(|option| {
+            let mut contents: Vec<Line> = vec![];
+
+            if let Some(direction) = &option.stage_direction {
+                let padded = format!("({})", direction);
+                let wrapped = textwrap::wrap(&padded, wrap_options.clone());
+                let wrapped_direction = wrapped
+                        .iter()
+                        .map(|x| { Line::from(x.to_string()).fg(style::Color::Yellow)});
+                contents.extend(wrapped_direction);
+            }
+
+            let wrapped = textwrap::wrap(&option.text, wrap_options.clone());
+            let text = wrapped
+                        .iter()
+                        .map(|x| { Line::from(x.to_string())});
+            contents.extend(text);
+            Text::from_iter(contents)
+        }).collect();
+        let list = List::new(options)
+            .block(borders)
+            .style(ratatui::style::Color::White)
+            .highlight_symbol("> ".fg(ratatui::style::Color::Cyan))
+            .highlight_style(style::Style::new().bold().bg(style::Color::DarkGray))
+            .repeat_highlight_symbol(true);
+        StatefulWidget::render(list, area, buf, state);
+    }
+}
+
+pub struct Conversation<'a>(pub &'a Vec<ConversationEntry>);
+
+impl Conversation<'_> {
+    fn format_line<'a>(entry: &'a ConversationEntry, max_width: usize) -> Text<'a> {
+        let prefix = entry.prefix().unwrap_or_default();
+
+        let style = entry.prefix_style();
+
+        let text_style = entry.text_style();
+
+        let avail_width = max_width - prefix.len();
+        let indent = " ".repeat(prefix.len());
+        let wrap_options = textwrap::Options::new(avail_width).initial_indent(prefix).subsequent_indent(&indent);
+        let wrapped: Vec<Line> = textwrap::wrap(&entry.to_string(), wrap_options)
+            .iter()
+            .enumerate()
+            .map(|(idx, s)| {
+                if idx == 0 {
+                    Line::from_iter([Span::from(prefix).style(style), Span::from(s[prefix.len()..].to_string()).style(text_style)])
+                } else {
+                    Line::from(s.to_string()).style(text_style)
+                }
+            }).collect();
+
+        Text::from_iter(wrapped)
+    }
+}
+
+impl<'a> StatefulWidget for Conversation<'a> {
+    fn render(self, area: Rect, buf: &mut ratatui::prelude::Buffer, state: &mut Self::State) {
+        let width = area.width.into();
+        let items: Vec<Text> = self.0.iter().rev().map(|entry| {
+            Self::format_line(entry, width)
+        }).collect();
+        // TODO: Would be nice to be able to scroll a longer conversation with the scroll wheel, or with page up/down
+        let list = List::new(items)
+            .block(Block::bordered().border_style(Color::LightYellow).title("Conversation (Press Tab to select and read Eva's lines aloud)"))
+            .direction(ListDirection::BottomToTop)
+            .highlight_symbol("> ")
+            .highlight_style(Style::new().bold().fg(Color::Cyan));
+        StatefulWidget::render(list, area, buf, state);
+    }
+    
+    type State = ListState;
+}
+
+pub struct Volume(pub f64, pub bool);
+
+impl Widget for Volume {
+    fn render(self, area: Rect, buf: &mut ratatui::prelude::Buffer)
+    where
+        Self: Sized
+    {
+        const NOISE_FLOOR: f64 = 50.;
+        let vu_pct = 1.0 - (self.0.abs().min(NOISE_FLOOR) / NOISE_FLOOR);
+
+        let volume_color = if self.1 {
+            if vu_pct >= 0.85 {
+                style::Color::Red
+            } else if vu_pct >= 0.60 {
+                style::Color::Yellow
+            } else {
+                style::Color::LightGreen
+            }
+        } else {
+            style::Color::Gray
+        };
+
+        let gauge = Gauge::default()
+            .ratio(vu_pct)
+            .use_unicode(true)
+            .gauge_style(volume_color)
+            .label(format!("{:.01}dB", self.0));
+        gauge.render(area, buf);
+    }
+}
+
+pub struct RecordingStatus(pub bool);
+
+impl Widget for RecordingStatus {
+    fn render(self, area: Rect, buf: &mut ratatui::prelude::Buffer)
+    where
+        Self: Sized
+    {
+        Line::from_iter(
+            if self.0 {
+                [
+                    Span::from(" "),
+                    Span::from(" Recording ").bg(style::Color::LightRed).fg(style::Color::White),
+                    Span::from(" ")
+                ]
+            } else {
+                [
+                    Span::from(" "),
+                    Span::from("  Stopped  ").bg(style::Color::DarkGray).fg(style::Color::White),
+                    Span::from(" ")
+                ]
+            }
+        ).render(area, buf);
+    }
+}
+
+pub struct StatusBar<'a>(pub &'a Scene);
+
+impl Widget for StatusBar<'_> {
+    fn render(self, area: Rect, buf: &mut ratatui::prelude::Buffer)
+    where
+        Self: Sized {
+        let time_remaining: Duration = self.0.direction().time_remaining();
+        let minutes_remaining = time_remaining.num_seconds() / 60;
+
+        let negative = time_remaining.abs() != time_remaining;
+
+        let time_style = if minutes_remaining <= 0 || negative {
+            Style::new().fg(ratatui::style::Color::LightRed).underlined()
+        } else if minutes_remaining < 5 {
+            Style::new().fg(ratatui::style::Color::LightRed).bold()
+        } else if minutes_remaining < 10 {
+            ratatui::style::Color::Red.into()
+        } else if minutes_remaining < 25 {
+            ratatui::style::Color::Yellow.into()
+        } else if minutes_remaining < 60 {
+            ratatui::style::Color::Green.into()
+        } else {
+            ratatui::style::Color::Blue.into()
+        };
+
+        let formatted_time = if negative {
+            format!("-{:0>2}:{:0>2}:{:0>2}", time_remaining.num_hours().abs(), time_remaining.num_minutes().abs()% 60, time_remaining.num_seconds().abs() % 60)
+        } else {
+            format!("{:0>2}:{:0>2}:{:0>2}", time_remaining.num_hours(), time_remaining.num_minutes() % 60, time_remaining.num_seconds() % 60)
+        };
+
+        let artifact_count = self.0.artifact_count;
+        let artifact_stats = self.0.artifact_stats;
+
+        let status_line = Line::from_iter([
+            Span::from(format!("Playlist: {}", self.0.direction().playlist)).style(ratatui::style::Color::LightBlue),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("{} tracks", self.0.playlist().len())).style(ratatui::style::Color::LightBlue),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("Time Remaining: {}", formatted_time)).style(time_style),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("{} ({}/{}/{}) artifacts recorded", artifact_count, artifact_stats.0, artifact_stats.1, artifact_stats.2)).style(ratatui::style::Color::LightBlue),
+            Span::from(" | ").style(ratatui::style::Color::DarkGray),
+            Span::from(format!("{} tokens sacrificed", self.0.tokens_consumed)).style(ratatui::style::Color::LightCyan),
+            // TODO: Should show the available and consumed context window in terms of tokens here
+        ]);
+        status_line.render(area, buf);
+    }
+}