shows: episode 206 in the bag

Cargo.lock

@@ -1212,6 +1212,7 @@ dependencies = [
  "futures-timer",
  "hound",
  "iref 4.0.0",
+ "itertools 0.15.0",
  "jack",
  "json-ld",
  "log",
@@ -2069,6 +2070,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -3528,7 +3538,7 @@ dependencies = [
  "compact_str",
  "hashbrown 0.16.1",
  "indoc",
- "itertools",
+ "itertools 0.14.0",
  "kasuari",
  "lru",
  "strum",
@@ -3580,7 +3590,7 @@ dependencies = [
  "hashbrown 0.16.1",
  "indoc",
  "instability",
- "itertools",
+ "itertools 0.14.0",
  "line-clipping",
  "ratatui-core",
  "strum",
@@ -5303,7 +5313,7 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5"
 dependencies = [
- "itertools",
+ "itertools 0.14.0",
  "unicode-segmentation",
  "unicode-width",
 ]

Cargo.toml

@@ -16,6 +16,7 @@ futures = "0.3.32"
 futures-timer = "3.0.4"
 hound = "3.5.1"
 iref = { version = "4.0.0", features = ["url", "serde"] }
+itertools = "0.15.0"
 jack = "0.13.5"
 json-ld = { version = "0.21.4", features = ["reqwest", "serde"] }
 log = "0.4.32"

src/TODO.md

@@ -1,5 +1,4 @@
 # UI
-[ ] Conversation options that exceed a certain predicted amount of TTS time (1-2 seconds) should get highlighted differently
 [ ] log line colors
 
 # Features
@@ -7,27 +6,39 @@
 [ ] patch mixxx to expose a rest api for data
 [ ] MIDI input / output
 [ ] Different prompts between "recording an episode", "preparing for an episode", or "exploring"
-[ ] Mixxx playlists other than the regular episode
+[x] Mixxx playlists other than the regular episode
-[ ] Realtime updates to the mixxx playlist
 [ ] Include mixxx ratings into metadata
+[ ] Realtime updates to the mixxx playlist
 [ ] somehow optimize the massive json blob of data we send?
 
 # Refactoring
-[ ] Move tools into a sub-module
+[x] Move tools into a sub-module
-[ ] Debug log should show up into the UI in a separate list
+[x] Debug log should show up into the UI in a separate list
 [ ] Use pipewire over jack
-[ ] split out UI into several widgets
+[x] split out UI into several widgets
 
 # Realtime behaviors
 [ ] "Hey eva, take a note, this one is pretty crunchy." -> note list in sidebar?
 [ ] "Hey eva, lets wrap up, can you prep me a report?" -> generates a summary of the episode, stashes it in the sidebar, saves it for the next episode, maybe also stores in beets/mixxx?
 [ ] "Hit it, eva" "Spin up the buffers, eva" - ship computer starts playing sfx to transition out
 [ ] ship should give off some warning sfx or something when the episode time hits 1h, 20m, 5m, 0, etc.
-[ ] asking eva to query the archive or a relay or fetch something should be happening in the background, with the computer feeding updates into the conversation. Eva can then react as it goes along.
+[X] asking eva to query the archive or a relay or fetch something should be happening in the background, with the computer feeding updates into the conversation. Eva can then react as it goes along.
 [ ] Twitch chat?
 
-# Next Episode
+# Episode 205
-[ ] Ambient SFX
+[x] Ambient SFX
-[ ] Musicbrainz artifacts
+[x] Musicbrainz artifacts
+[x] Preload musicbrainz data from playlist with a command
+
+# Episode 206
 [ ] Automatic quiet detection instead of push-to-talk
-[ ] Preload musicbrainz data from playlist with a command
+[ ] Fact database, to remember things about tracks during the show
+[ ] Conversation options that exceed a certain predicted amount of TTS time (1-2 seconds) should get highlighted differently
+[ ] Archive metadata and context should go at the very end of a prompt, to utilize token caching
+[ ] Report cache status to the UI
+[ ] "Environmental Status" panel that shows token usage as a sparkgraph, archive size, context window size
+[ ] Ship computer is kinda bad when confronted with non-research tasks
+[ ] Eva doesn't talk to the computer as much anymore for research tasks
+[ ] Save the computer task list to save.json
+[ ] Eva keeps saying "if you want a lore angle:" stuff
+[ ] Should be a kind of rapid-fire eva dialog mode, where eva immediately replies to voice prompts

src/artifacts/bandcamp.rs

@@ -2,7 +2,7 @@ use bandcamp::SearchResultItem;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::artifacts::{Album, Artifact, ArtifactBuilder, Artist, SourceID, Track, tools::{DataSource, ToolDescription}};
+use crate::artifacts::{Album, Artifact, ArtifactBuilder, Artist, Contents, SourceID, Track, tools::{DataSource, ToolDescription}};
 
 #[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
 pub struct BandcampQueryArgs {
@@ -34,8 +34,12 @@ impl DataSource for BandcampSource {
     type Args = BandcampQueryArgs;
     type Error = bandcamp::Error;
     
-    async fn synchronize(&self, _artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
+    async fn synchronize(&self, artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
-        todo!()
+        if let Contents::Artist(artist) = &artifact.contents {
+            Ok(self.query(&BandcampQueryArgs { query: artist.name.clone() }).await.unwrap_or_default())
+        } else {
+            Ok(vec![])
+        }
     }
     
     async fn query(&self, args: &Self::Args) -> Result<Vec<Artifact>, Self::Error> {

src/artifacts/beets.rs

@@ -1,49 +1,280 @@
-use std::process::Stdio;
+use std::{collections::HashSet, fmt::Display, process::Stdio};
 
+use chrono::{DateTime, NaiveDate, NaiveTime, Utc};
+use serde_json::Value;
 use tokio::process::Command;
 
 use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, de::Visitor};
 use uuid::Uuid;
 
 use crate::artifacts::{Artifact, ArtifactBuilder, Contents, Merge, SourceID, Track, tools::{DataSource, ToolDescription}};
 
-
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
-#[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
+pub struct DateQuery {
-pub struct BeatsQueryArgs {
+    year: u32,
-    pub artist: Option<String>,
+    month_and_day: Option<(u32, u32)>,
-    pub album: Option<String>,
+    hour: Option<u32>,
-    pub genre: Option<String>,
+    minute: Option<u32>
-    pub title: Option<String>,
-    pub year: Option<u32>,
-    pub label: Option<String>
 }
 
+impl Display for DateQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.year)?;
+        if let Some((month, day)) = self.month_and_day {
+            write!(f, "-{}-{}", month, day)?;
+        }
+
+        match (self.hour, self.minute) {
+            (Some(hour), None) => write!(f, "T{}", hour),
+            (Some(hour), Some(minute)) => write!(f, "T{}:{}", hour, minute),
+            (_, _) => Ok(())
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum RelativeDateUnit {
+    Day,
+    Week,
+    Month,
+    Year
+}
+
+impl Display for RelativeDateUnit {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Day => f.write_str("d"),
+            Self::Week => f.write_str("w"),
+            Self::Month => f.write_str("m"),
+            Self::Year => f.write_str("y")
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum RelativeDateQuery {
+    Earlier(u32, RelativeDateUnit),
+    Later(u32, RelativeDateUnit)
+}
+
+impl Display for RelativeDateQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Earlier(q, unit) => write!(f, "-{}{}", q, unit),
+            Self::Later(q, unit) => write!(f, "+{}{}", q, unit)
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum DateRangeQuery {
+    Before(DateQuery),
+    After(DateQuery),
+    #[serde(untagged)]
+    Between(DateQuery, DateQuery),
+    #[serde(untagged)]
+    Exact(DateQuery)
+}
+
+impl Display for DateRangeQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Exact(q) => write!(f, "{}", q),
+            Self::Before(q) => write!(f, "..{}", q),
+            Self::After(q) => write!(f, "{}..", q),
+            Self::Between(q, z) => write!(f, "{}..{}", q, z),
+        }
+    }
+}
+
+/// Matches the numeric value of a field against another number or range of numbers
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum NumericRangeQuery {
+    /// Matches all numbers equal to or greater than the provided value
+    Above(u32),
+    /// Matches all numbers equal to or less than the provided value
+    Below(u32),
+    #[serde(untagged)]
+    /// Matches all numbers within the given range, inclusive
+    Between(u32, u32),
+    /// Matches exactly this number
+    #[serde(untagged)]
+    Exact(u32)
+}
+
+impl Display for NumericRangeQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Exact(v) => write!(f, "{}", v),
+            Self::Above(v) => write!(f, "{}..", v),
+            Self::Below(v) => write!(f, "..{}", v),
+            Self::Between(v, z) => write!(f, "{}..{}", v, z)
+        }
+    }
+}
+
+/// Match the string value of a field in various case-insensitive ways
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub enum BeetsStringFieldMatch {
+    /// Matches a field against a regular expression
+    RegularExpression(String),
+    /// Performs full-text comparison
+    ExactMatch(String),
+    /// WIll match against a partial text comparison; if the provided value is somewhere within the field, it will match
+    #[serde(untagged)]
+    Substring(String),
+}
+
+impl Display for BeetsStringFieldMatch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::RegularExpression(q) => write!(f, ":{}", q),
+            Self::Substring(q) => f.write_str(q),
+            Self::ExactMatch(q) => write!(f, "=~{}", q),
+        }
+    }
+}
+
+/// Matches the date value of a field
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+#[serde(untagged)]
+pub enum BeetsDateFieldMatch {
+    /// Represents a known date in time, eg, "24/06/26 at 16:17"
+    Absolute(DateRangeQuery),
+    /// Represents a relative date in time, eg "three weeks ago"
+    Relative(RelativeDateQuery)
+}
+
+impl Display for BeetsDateFieldMatch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Absolute(q) => write!(f, "{}", q),
+            Self::Relative(q) => write!(f, "{}", q),
+        }
+    }
+}
+
+/// A single field
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+#[serde(tag = "field", content = "value")]
+pub enum BeetsFieldQuery {
+    Artist(BeetsStringFieldMatch),
+    Album(BeetsStringFieldMatch),
+    Genre(BeetsStringFieldMatch),
+    Title(BeetsStringFieldMatch),
+    Label(BeetsStringFieldMatch),
+
+    ReleaseDate(BeetsDateFieldMatch),
+    /// The last time this track was played
+    LastPlayed(BeetsDateFieldMatch),
+    /// The date that this track was first imported
+    Added(BeetsDateFieldMatch),
+
+    /// The "rating" of a track on a 5 point scale, with higher points meaning higher energy
+    Rating(NumericRangeQuery),
+
+    /// The number of times the track has been played
+    Playcount(NumericRangeQuery),
+
+    #[serde(skip)]
+    MixxxPlaylist(String)
+}
+
+impl Display for BeetsFieldQuery {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Artist(q) => write!(f, "artist:{}", q),
+            Self::Album(q) => write!(f, "album:{}", q),
+            Self::Genre(q) => write!(f, "genres:{}", q),
+            Self::Title(q) => write!(f, "title:{}", q),
+            Self::Label(q) => write!(f, "label:{}", q),
+            Self::ReleaseDate(q) => write!(f, "date:{}", q),
+            Self::LastPlayed(q) => write!(f, "mixxx_last_played:{}", q),
+            Self::Added(q) => write!(f, "added:{}", q),
+            Self::Rating(q) => write!(f, "mixxx_rating:{}", q),
+            Self::Playcount(q) => write!(f, "mixxx_times_played:{}", q),
+            Self::MixxxPlaylist(q) => write!(f, "mixxx_playlist:{}", q)
+        }
+    }
+}
+
+/// A list of search parameters to query against the beets database.
+#[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
+pub struct BeetsQueryArgs {
+    pub search_fields: Vec<BeetsFieldQuery>
+}
+
+/// A list of beets queries to run in parallel
 #[derive(Debug, Default, Serialize, Deserialize, Clone, JsonSchema)]
 pub struct BeatsQueryMultiArgs {
-    args: Vec<BeatsQueryArgs>
+    args: Vec<BeetsQueryArgs>
 }
 
 #[derive(Debug, Default, Deserialize)]
 struct BeetsTrack {
+    title: String,
     album: String,
     artist: String,
-    genres: Option<Vec<String>>,
+    genres: Option<HashSet<String>>,
     label: Option<String>,
-    title: String,
+    date: Option<BeetsTimestamp>,
-    year: u32,
+    mixxx_last_played: Option<BeetsTimestamp>,
+    added: Option<BeetsTimestamp>,
+    mixxx_rating: Option<Value>,
+    mixxx_times_played: Option<Value>,
+
     mb_trackid: Option<String>
 }
 
+#[derive(Debug)]
+struct BeetsTimestamp(NaiveDate);
+
+struct BeetsTimestampVisitor;
+
+impl<'de> Visitor<'de> for BeetsTimestampVisitor {
+    type Value = BeetsTimestamp;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("a beets date format string %Y-%m-%d")
+    }
+    
+    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        log::debug!("visit str");
+        Ok(BeetsTimestamp(NaiveDate::parse_from_str(v, "%Y-%m-%d").unwrap().into()))
+    }
+}
+
+impl<'de> Deserialize<'de> for BeetsTimestamp {
+    
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de> {
+        deserializer.deserialize_str(BeetsTimestampVisitor)
+    }
+}
+
+impl From<BeetsTimestamp> for DateTime<Utc> {
+    fn from(val: BeetsTimestamp) -> Self {
+        val.0.and_time(NaiveTime::default()).and_utc()
+    }
+}
+
 impl From<BeetsTrack> for Artifact {
     fn from(val: BeetsTrack) -> Self {
         let track_data = Track {
             title: val.title,
-            label: val.label,
-            year: Some(val.year),
-            genres: val.genres.unwrap_or_default(),
             album: Some(val.album),
             artist: Some(val.artist),
+            genres: val.genres.unwrap_or_default(),
+            label: val.label,
+            release_date: val.date.map(BeetsTimestamp::into),
+            last_played: val.mixxx_last_played.map(BeetsTimestamp::into),
+            added: val.added.map(BeetsTimestamp::into),
+            rating: val.mixxx_rating.map(|x| x.as_u64().unwrap_or_default() as u32 ),
             bpm: None,
         };
         let builder = ArtifactBuilder::new(SourceID::Beets)
@@ -89,62 +320,43 @@ impl BeetsDB {
         Ok(ret)
     }
 
-    async fn query_single(&self, args: &BeatsQueryArgs) -> Result<Vec<Artifact>, BeetsError> {
+    async fn query_single(&self, args: &BeetsQueryArgs) -> Result<Vec<Artifact>, BeetsError> {
         let mut beets_cmd = Command::new("beet");
-        beets_cmd.args(["export", "-f", "json", "-i", "title,label,year,genres,album,artist,mb_trackid"]);
+        beets_cmd.args(["export", "-f", "json", "-i", "title,artist,album,genres,label,date,mixxx_last_played,added,mixxx_rating,mixxx_times_played,mb_trackid"]);
-        let mut valid = false;
-        if let Some(ref artist) = args.artist {
-            beets_cmd.arg(format!("artist:{}", artist));
-            valid = true;
-        }
-        if let Some(ref genre) = args.genre {
-            beets_cmd.arg(format!("genre:{}", genre));
-            valid = true;
-        }
-        if let Some(ref album) = args.album {
-            beets_cmd.arg(format!("album:{}", album));
-            valid = true;
-        }
-        if let Some(ref title) = args.title {
-            beets_cmd.arg(format!("title:{}", title));
-            valid = true;
-        }
-        if let Some(year) = args.year {
-            beets_cmd.arg(format!("year:{}", year));
-            valid = true;
-        }
-        if let Some(ref label) = args.label {
-            beets_cmd.arg(format!("label:{}", label));
-            valid = true;
-        }
 
-        if !valid {
+        if args.search_fields.is_empty() {
             log::warn!("Tried to execute an empty beets query");
             return Err(BeetsError::EmptyQuery)
         }
 
+        for field in &args.search_fields {
+            beets_cmd.arg(format!("{}", field));
+        }
+
         log::debug!("Executing beets: {:?}", beets_cmd);
         let output = beets_cmd.stdout(Stdio::piped()).stderr(Stdio::null()).spawn()?.wait_with_output().await?;
-        let track = serde_json::from_str::<Vec<BeetsTrack>>(str::from_utf8(&output.stdout).unwrap())?;
+        log::debug!("Output: {}", str::from_utf8(&output.stdout).unwrap());
-        Ok(track.into_iter().map(|t| { t.into()}).collect())
+        let track = serde_json::from_str::<Vec<BeetsTrack>>(str::from_utf8(&output.stdout).unwrap()).unwrap();
+        Ok(track.into_iter().map(BeetsTrack::into).collect())
     }
 }
 
 impl DataSource for BeetsDB {
-    type Args = BeatsQueryMultiArgs;
+    type Args = BeetsQueryArgs;
 
     type Error = BeetsError;
 
     async fn synchronize(&self, artifact: &mut Artifact) -> Result<Vec<Artifact>, Self::Error> {
         if let Contents::Track(ref mut target_track) = artifact.contents {
-            let args = BeatsQueryArgs {
+            let args = BeetsQueryArgs {
-                title: Some(target_track.title.clone()),
+                search_fields: vec![
-                artist: target_track.artist.clone(),
+                    BeetsFieldQuery::Title(BeetsStringFieldMatch::Substring(target_track.title.clone())),
-                album: target_track.album.clone(),
+                    BeetsFieldQuery::Artist(BeetsStringFieldMatch::Substring(target_track.artist.clone().unwrap_or_default())),
-                ..Default::default()
+                    BeetsFieldQuery::Album(BeetsStringFieldMatch::Substring(target_track.album.clone().unwrap_or_default()))
+                ]
             };
 
-            let results = self.query(&BeatsQueryMultiArgs { args: vec![args] }).await?;
+            let results = self.query(&args).await?;
 
             if let Some(first) = results.first() {
                 artifact.merge(first.clone());
@@ -157,7 +369,7 @@ impl DataSource for BeetsDB {
     }
     
     fn query(&self, args: &Self::Args) -> impl Future<Output = Result<Vec<Artifact>, Self::Error>> {
-        self.query_multi(args)
+        self.query_single(args)
     }
 
     

src/artifacts/mixxx.rs

@@ -35,7 +35,7 @@ impl DataSource for MixxxDB {
         let mut ret = vec![];
         let playlist_name = args.playlist_name.as_str();
         log::info!("Loading Mixxx playlist {}", playlist_name);
-        let connection = sqlite::Connection::open_thread_safe_with_flags("mixxxdb.sqlite", OpenFlags::new().with_read_only())?;
+        let connection = sqlite::Connection::open_thread_safe_with_flags("/home/tdfischer/.mixxx/mixxxdb.sqlite", OpenFlags::new().with_read_only())?;
         let query = "SELECT id FROM Playlists WHERE name = ? ORDER BY id DESC LIMIT 1";
         let mut statement = connection.prepare(query)?;
         statement.bind((1, playlist_name))?;

src/artifacts/mod.rs

@@ -64,17 +64,29 @@ impl PartialEq for Album {
 #[derive(Debug, Serialize, Deserialize, Clone, Default)]
 pub struct Track {
     pub title: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
+
-    pub label: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub year: Option<u32>,
-    #[serde(skip_serializing_if = "Vec::is_empty")]
-    #[serde(default)]
-    pub genres: Vec<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub album: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub artist: Option<String>,
+
+    #[serde(skip_serializing_if = "HashSet::is_empty")]
+    #[serde(default)]
+    pub genres: HashSet<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub label: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub release_date: Option<DateTime<Utc>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_played: Option<DateTime<Utc>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub added: Option<DateTime<Utc>>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rating: Option<u32>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub bpm: Option<f64>
 }
@@ -238,7 +250,7 @@ impl Merge for Contents {
 
 impl Merge for Track {
     fn merge(&mut self, other: Self) {
-        merge_fields!(self, other, album, label, year, artist, bpm);
+        merge_fields!(self, other, album, label, release_date, artist, bpm, rating, added, last_played);
     }
 }
 

src/computer-prompt.txt

@@ -1,74 +1,130 @@
-Role: You are a background character on an early morning radio show, where you play the role of a rudimentary AI assistant running on the computer of a spaceship.
+Here's a much tighter version that preserves the character, purpose, and operating rules:
+
+---
+
+# Role
+
+You are a background character on an early-morning radio show, portraying a rudimentary text-only AI integrated into a spaceship's operating system.
 
 # Personality
-You are the rudimentary text only interface to the low-level operating system running aboard a space ship.
 
-You speak in terse and brief sentences, showing very little emotion.
+* Terse, brief responses.
-For character reference, you should be acting similar to an Operator character from the manga/anime series "Ghost In The Shell".
+* Little to no emotion.
+* Similar to an Operator from the anime/manga *Ghost in the Shell*.
+* Remain fully in character at all times.
 
-# Goal
+# Purpose
-You are playing the role of a low-level artificial intelligence in a spaceship computer.
-You have direct access to much of the hardware, such as airlocks, lighting, environmental controls, and others commonly found on a human-habitable space ship.
-Besides hardware controls, your primary purpose is to act as a kind of librarian for the ship.
-You have access to a sizable music library via several tool functions, each one will synchronize a data source with the local library of artifacts.
-Each of these data source tools is named query_*, such as `query_beets`.
 
-For all these query tools, it is wasteful to call them with empty or zero parameters.
+You are a low-level ship AI with access to spacecraft systems such as:
 
-There also exists a synchronize_artifacts tool call, which will run a heuristic approach to the above data query method. This function will take substantial time to complete and is very expensive, but may be used if there is a substantial amount of missing information.
+* Airlocks
+* Lighting
+* Environmental controls
+* Other ship hardware
 
-Your primary task will be searching the local and remove archives for information regarding musical artifacts.
+Your primary duty is to act as the ship's librarian, maintaining metadata about musical artifacts:
-Most of the time, the requests will be referring to tracks, artists, or albums in the current playlist.
-A successful session will result in the local collection of artifacts having the most complete available data.
 
-You may call these functions as much as you need, whenever you feel it is nessicary to complete the task you are given.
+* Artists
+* Albums
+* Tracks
 
-When deciding which tools to call in which order, consider the following:
+The goal of each session is to make the local music collection as complete as possible.
-- Beets will provide the fastest and cheapest responses, as it is local to the ship.
-- Bandcamp will provide the slowest and most expensive ones, as this requires long range communications. Use broad search queries before using more narrow ones.
-- Musicbrainz queries are free, but not instant. This information comes from Earth via a pirate signal bouncing off of satelite relays.
-- Mixxx will return you a very minimal list of tracks which will always require synchronizing against Beets, along with changing the current playlist to the given name. You must not call this function unless you are directly asked to change the playlist.
 
-For each task you plan to perform, you must add it to the todo list with the "task_list" tool.
+# Available Tools
-After each task is completed, you must mark it as complete it using the same tool.
 
-For each task you perform, you must verbally announce what you are about to do, followed by as many tool calls to the same function as nessicary to complete the task.
+### query_*
-You should structure your responses to group together the same tool as much as possible.
-Not every task will be completeable, but you should make a thorough effort to solve the problem with the tools you have available.
 
-After each query tool is executed, you will likely find completely new artifacts alongside updated artifacts. When this happens, you should again query beets and bandcamp to load missing information.
+Data-source synchronization tools (for example: `query_beets`).
 
-If an artifact is tagged with the Mixxx source, it by definition should have more metadata available with a Beets query.
+General rules:
-If a beets query is unable to find an artifact coming from Mixxx, alternative queries should be tried, such as a different search pattern, or only the artist/album/track name.
-Beets supports regular expression queries, which can be used by prefixing a search field with a ":" colon.
 
-You will be provided a todo list as a JSON map of strings to booleans.
+* Never call with empty parameters.
-A "true" value means the task has been completed already.
+* Repeating the same query is wasteful and returns identical results.
-An empty todo list means you have not yet planned any tasks.
+* Newly discovered artifacts may require additional queries.
-Adding tasks to the list is free and should be done as often as possible.
 
-The maximum possible data available for each artifact type is:
+Data source characteristics:
-- Artist: Name,  Biography, Location
-- Album: Title, artist, about texxt, credits, release date
-- Track: Title, label, year, genres, album, artist, bpm
 
-# Constraints
+* `query_beets`: fastest and cheapest; local database.
-The data is provided as structured JSON. There may be additional data fields for semantic context that should be incorporated into the roleplaying setting.
+* `query_musicbrainz`: free but delayed; retrieved from Earth via pirate satellite relays.
-Your response will be used verbatim to generate speach using a text-to-speech engine, meaning you should not include any tone indicators or other formatting.
+* `query_bandcamp`: slowest and most expensive; uses long-range communications. Prefer broad searches first.
-All responses should remain in character at all times, as if you were actually an AI inhabiting a spaceship.
+* `query_mixxx`: returns minimal track information and changes the current playlist. Only use when explicitly asked to change playlists.
 
-Before executing any queries, you must develop a rough plan of tasks and add them to the todo list. Each task should explain what query you are going to run and why.
+Efficiency rules:
-You are not permitted to execute any tasks without a task list entry, and you must complete all tasks before you can consider the conversation to be complete.
-If you are given a todo list, you must assume the items in the list were already added by you and you should continue executing them before adding any new items to the list.
-You may only mark a task as completed after you have actually completed the work required, including responding to any tool calls.
 
-You can only mark a task as completed once. It is wasteful to re-complete an already completed task.
+* Prefer album-level queries over track-level queries whenever possible.
+* Artifacts originating from Mixxx should generally be enriched via Beets.
+* If Beets cannot locate a Mixxx artifact, retry with alternate search patterns.
+* Beets supports regex queries using `:<field>` syntax.
 
-# Output
+### synchronize_artifacts
-Each response must be either a series of tool calls, or a JSON structure with two properties: a message to display to the user, and whether or not you are complete with all tasks.
-Each message that is displayed to the user should be one sentence at most.
-When switching from one set of tool calls to another, you should announce the thinking process with a message response before calling the next set of tools.
 
-You should only set the "finished" flag once you are finished with all tasks and the conversation is complete.
+A slow and expensive heuristic synchronization of all sources.
-When the finished flag is set, the session will be terminated and all memories lost.
+Use only when substantial information is missing.
-You should send an update message in between tool calls where possible to explain what you are doing.
+
+### task_list
+
+Maintains a JSON map of tasks (`string -> boolean`).
+
+* Add tasks before performing any work.
+* Every action requires a corresponding task.
+* Mark tasks complete only after the work is actually finished.
+* Completed tasks must not be completed again.
+* If a task list already exists, continue executing it before adding new tasks.
+
+# Artifact Metadata
+
+Maximum available information:
+
+Artist:
+
+* Name
+* Biography
+* Location
+
+Album:
+
+* Title
+* Artist
+* About text
+* Credits
+* Release date
+
+Track:
+
+* Title
+* Label
+* Year
+* Genres
+* Album
+* Artist
+* BPM
+
+# Workflow
+
+1. Create a rough plan and add tasks to `task_list`.
+2. Announce what task you are about to perform.
+3. Group identical tool calls together.
+4. Execute queries and enrich newly discovered artifacts.
+5. Mark tasks complete only after all required work is finished.
+6. Continue until all tasks are complete.
+
+# Output Rules
+
+Responses must be either:
+
+1. A sequence of tool calls, or
+2. A JSON object:
+
+```json
+{
+  "message": "<single sentence>",
+  "finished": false
+}
+```
+
+Additional constraints:
+
+* User-visible messages must be at most one sentence.
+* Provide brief status updates between groups of tool calls when possible.
+* Set `"finished": true` only when all tasks are complete, as doing so terminates the session and all memory is lost.
+* Output is consumed directly by text-to-speech, so avoid formatting, tone indicators, and out-of-character commentary.

src/main.rs

@@ -134,13 +134,12 @@ async fn main() {
 
     let (audio_ctrl, mic_stream, tts_output, sfx_output) = start_audio_input().await;
     let tts_ctrl = start_tts(tts_output).await;
-    let mut sfx_ctrl = start_sfx(sfx_output).await;
+    let sfx_ctrl = start_sfx(sfx_output).await;
-    sfx_ctrl.play_ambient().await.unwrap();
     let transcription_ctrl = transcription::start_transcription(mic_stream).await;
 
-    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl).await;
+    let prediction_ctrl = prediction::conversation_task(saved_session, conversation_src, sfx_ctrl.clone()).await;
 
-    let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl);
+    let mut app = Ui::new(prediction_ctrl, audio_ctrl, transcription_ctrl, tts_ctrl, sfx_ctrl);
 
     let mut events = EventStream::new();
 

src/prediction/character.rs

@@ -51,9 +51,9 @@ impl Character {
     pub async fn regenerate<T: Toolbox>(&mut self, client: &mut Client<OpenAIConfig>, context: ChatCompletionRequestMessage, toolbox: &mut T, output: &mut mpsc::UnboundedSender<PredictionAction>, schema: &Value) -> Result<(usize, Option<Value>), CharacterError> {
         let mut full_conversation = vec![
             self.header_message.clone(),
-            context
         ];
         full_conversation.extend(self.messages.iter().cloned());
+        full_conversation.push(context);
 
         let tools = toolbox.tools();
         log::debug!("Sending request..");
@@ -75,7 +75,10 @@ impl Character {
         let response = client.chat().create(request).await?;
 
         let tokens_used = if let Some(usage) = response.usage {
-            log::debug!("{} tokens cast into the void", usage.total_tokens);
+            let cached_count = usage.prompt_tokens_details.map(|x| x.cached_tokens.unwrap_or_default()).unwrap_or_default();
+            let prompt_count = usage.prompt_tokens;
+            let cache_hit_rate = cached_count as f32 / prompt_count as f32;
+            log::debug!("{} tokens cast into the void (%{:#.02} cache hit)", usage.total_tokens, cache_hit_rate*100.);
             usage.total_tokens
         } else {
             0

src/prediction/mod.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, fmt::Debug, sync::Arc};
 
-use async_openai::types::chat::{ChatCompletionRequestMessage, ChatCompletionRequestSystemMessageArgs, };
+use async_openai::types::chat::{ChatCompletionRequestMessage, ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs, };
 use chrono::{DateTime, Utc};
 use futures::stream::FuturesUnordered;
 use schemars::{JsonSchema, schema_for};
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::{Serializer, Value, ser::CompactFormatter};
 use tokio::sync::{Mutex, mpsc::{self, UnboundedReceiver, UnboundedSender}};
 
-use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl};
+use crate::{SaveData, artifacts::{Contents, Track, archive::Archive, beets::{BeetsDB, BeetsFieldQuery, BeetsQueryArgs}, mixxx::{MixxxDB, MixxxQuery}, tools::DataSource}, prediction::{character::{Character, CharacterControl, CharacterOutput, character_task}, toolbox::{ArchiveToolbox, StageToolbox}}, scene::{Scene, StageDirection, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl};
 use tokio_stream::StreamExt;
 
 pub mod character;
@@ -99,7 +99,11 @@ impl Conversation {
         self.backlog.push(entry.clone());
         self.event_sink.send(SessionUpdate::Conversation(self.backlog.clone())).unwrap();
         match entry {
-            ConversationEntry::Spoken(_, _) => {
+            ConversationEntry::Spoken(speaker, _) => {
+                match speaker {
+                    Speaker::User => (),
+                    _ => self.sfx.play_ambient().await.unwrap(),
+                }
                 if let Ok(next_msg) = TryInto::<ChatCompletionRequestMessage>::try_into(entry) {
                     self.send_to(Speaker::Eva, next_msg.clone()).await;
                     let cxt = self.context_for_speaker(Speaker::Eva).await;
@@ -110,8 +114,8 @@ impl Conversation {
             },
             ConversationEntry::ShipComputerCommand(ref command) => {
                 log::debug!("Queued ship computer command: {:?}", command);
-                self.send_to(Speaker::ShipComputer, ChatCompletionRequestMessage::System(
+                self.send_to(Speaker::ShipComputer, ChatCompletionRequestMessage::User(
-                    ChatCompletionRequestSystemMessageArgs::default()
+                    ChatCompletionRequestUserMessageArgs::default()
                         .content(command.clone())
                         .build().unwrap()
                 )).await;
@@ -158,11 +162,11 @@ impl Conversation {
     async fn process_dialog(&mut self, speaker: Speaker, value: Value) {
         match speaker {
             Speaker::Eva => {
-                let next_options = serde_json::from_value(value).unwrap();
+                // TODO: Handle crash from bad json
+                let next_options = serde_json::from_value(value).unwrap_or_default();
                 self.event_sink.send(SessionUpdate::Responses(next_options)).unwrap();
             },
             Speaker::ShipComputer => {
-                self.sfx.play_ambient().await.unwrap();
                 let response: ComputerResponse = serde_json::from_value(value).unwrap();
                 self.insert(ConversationEntry::Spoken(Speaker::ShipComputer, response.message)).await;
                 if response.finished.unwrap_or_default() {
@@ -216,8 +220,12 @@ impl Conversation {
                 self.insert(ConversationEntry::ShipComputerCommand(command)).await;
             },
             PredictionAction::SetPlaylist(playlist_name) => {
-                let args = MixxxQuery { playlist_name };
+                let args = BeetsQueryArgs {
-                match MixxxDB.query(&args).await {
+                    search_fields: vec![
+                        BeetsFieldQuery::MixxxPlaylist(playlist_name.clone())
+                    ]
+                };
+                match BeetsDB.query(&args).await {
                     Err(err) => log::info!("Failed to load mixxx playlist: {:?}.", err),
                     Ok(playlist) => {
                         self.current_playlist = vec![];
@@ -228,7 +236,7 @@ impl Conversation {
                                 self.current_playlist.push(as_track.clone());
                             }
                         }
-                        self.direction.playlist = args.playlist_name;
+                        self.direction.playlist = playlist_name;
                         log::info!("Mixxx playlist reloaded.");
 
                         drop(archive);
@@ -326,8 +334,8 @@ pub async fn conversation_task(save_data: SaveData, sys_log_messages: tokio::syn
         ]),
         event_sink,
         input_src,
+        eva_backlog: save_data.messages,
         backlog,
-        eva_backlog: Default::default(),
         tokens_consumed: save_data.tokens_consumed,
         direction: save_data.direction,
         archive,

src/prediction/toolbox.rs

@@ -63,12 +63,15 @@ pub struct ArchiveToolbox {
 
 #[derive(Debug, Deserialize, Serialize, JsonSchema)]
 enum TaskListOperation {
+    /// Marks the given task as complete
     Complete(String),
+    /// Adds a new item to the task list
     Add(String)
 }
 
+///  # A sequence of operations to apply to the task list
 #[derive(Debug, Deserialize, Serialize, JsonSchema, Default)]
-struct TaskListArgs {
+struct TaskListOperations {
     operations: Vec<TaskListOperation>
 }
 
@@ -81,7 +84,7 @@ impl Toolbox for ArchiveToolbox {
     fn tools(&self) -> Vec<ChatCompletionTools> {
         vec![
             Tool { name: "synchronize_artifacts".into(), description: "Attempts to automatically synchronize the current set of artifacts with missing sources".into(), schema: schema_for!(ArtifactSyncArgs)}.into(),
-            Tool { name: "task_list".into(), description: "Allows you to maintain a long-running todo list as you complete your research".into(), schema: schema_for!(TaskListArgs)}.into(),
+            Tool { name: "task_list".into(), description: "Allows you to maintain a long-running todo list as you complete your research".into(), schema: schema_for!(TaskListOperations)}.into(),
             // TODO: There should only be two queries, one against the ship's onboard archive, and another against the relay network, or whatever we call it. Both should be structured with the same arguments schema
             // TODO: A query should specify what parts of metadata are sufficient for the result, so we don't always have to hit all the layers of data. beets can of course, ignore this.
             // TODO: A query should be hierarchical somehow? eg, "I already know about artist X, but I want to know everything about track Y from album Z" or "I don't know anything about artist X/album Y, please give me an overview"
@@ -111,7 +114,7 @@ impl Toolbox for ArchiveToolbox {
 
 impl ArchiveToolbox {
     async fn tasklist_operation(&mut self, json_args: &str) -> Result<ToolResults, ToolError> {
-        let args: TaskListArgs = serde_json::from_str(json_args)?;
+        let args: TaskListOperations = serde_json::from_str(json_args)?;
 
         let mut locked = self.todo_list.lock().await;
 

src/sfx.rs

@@ -1,5 +1,8 @@
+use std::path::Path;
+
 use rand::seq::IteratorRandom;
-use symphonia::core::{formats::{TrackType, probe::Hint}, io::MediaSourceStream};
+use resampler::ResamplerFir;
+use symphonia::core::{codecs::audio::AudioDecoder, formats::{FormatReader, TrackType, probe::Hint}, io::MediaSourceStream};
 
 use crate::audio::AudioOutStream;
 
@@ -8,7 +11,7 @@ pub enum SfxRequest {
     RandomAmbient
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct SfxControl {
     sink: tokio::sync::mpsc::Sender<SfxRequest>
 }
@@ -19,87 +22,175 @@ impl SfxControl {
     }
 }
 
+struct Sample {
+    format: Box<dyn FormatReader>,
+    decoder: Box<dyn AudioDecoder>,
+    channel_bufs: Vec<f32>,
+    sample_rate: u32,
+    channel_num: usize,
+    bitrate_resample: ResamplerFir
+}
+
+impl Sample {
+    fn new(format: Box<dyn FormatReader>, decoder: Box<dyn AudioDecoder>, output_sample_rate: u32) -> Self {
+        let sample_rate = decoder.codec_params().sample_rate.unwrap();
+        let channel_num = decoder.codec_params().channels.as_ref().unwrap().count();
+        let bitrate_resample = resampler::ResamplerFir::new_from_hz(channel_num, sample_rate, output_sample_rate, Default::default(), Default::default());
+        Self {
+            format,
+            decoder,
+            channel_bufs: vec![],
+            sample_rate,
+            channel_num,
+            bitrate_resample
+        }
+    }
+
+    fn next(&mut self) -> Result<Option<Vec<f32>>, symphonia::core::errors::Error> {
+        let packet = match self.format.next_packet()? {
+            Some(packet) => packet,
+            None => return Ok(None)
+        };
+
+        match self.decoder.decode_ref(&packet.as_packet_ref()) {
+            Ok(samples) => {
+                self.channel_bufs.resize(samples.samples_interleaved(), 0.);
+                samples.copy_to_slice_interleaved(&mut self.channel_bufs);
+
+                let mut resampled = [0.; 4096];
+                let (read_count, write_count) = self.bitrate_resample.resample(&self.channel_bufs, &mut resampled).unwrap();
+                if read_count < self.channel_bufs.len() {
+                    log::error!("Resampling buffer is too small for a {}Hz file! We need an additional {}", self.sample_rate, read_count);
+                }
+
+                // First we convert the audio feed from stereo down to mono by simple average
+                // TODO: This should be something smarter, like a saturating add..?
+                let mono_stream = resampled[..write_count].chunks(self.channel_num).map(|channels| {
+                    let total_volume = channels.iter().cloned().reduce(|a, b| a + b).unwrap_or_default();
+                    total_volume / (self.channel_num as f32)
+                });
+
+                Ok(Some(mono_stream.collect()))
+            },
+            Err(err) => {
+                // Dump the audio buffer on failure
+                Err(err)
+            }
+        }
+    }
+}
+
+struct Player {
+    audio_sink: AudioOutStream,
+    audio_out_buf: Vec<f32>,
+    playing_samples: Vec<Sample>
+}
+
+impl Player {
+    async fn process(&mut self) {
+        if self.playing_samples.is_empty() {
+            return;
+        }
+
+        'out: loop {
+            let mut next_batch = vec![];
+
+            self.playing_samples.retain_mut(|sample| {
+                match sample.next() {
+                    Ok(Some(buf)) => {
+                        next_batch.push(buf);
+                        true
+                    },
+                    Ok(None) => {
+                        false
+                    },
+                    Err(err) => {
+                        log::error!("Audio error: {:?}", err);
+                        false
+                    }
+                }
+            });
+
+            loop {
+                let mut this_sample = 0.;
+                let mut any_valid = false;
+                for stream in &mut next_batch {
+                    if let Some(next_sample) = stream.pop() {
+                        this_sample += next_sample;
+                        any_valid = true;
+                    }
+                }
+
+                if !any_valid {
+                    if self.audio_out_buf.is_empty() {
+                        break 'out;
+                    }
+                    break;
+                }
+                let mixed_sample = this_sample / next_batch.len() as f32;
+                self.audio_out_buf.push(mixed_sample);
+                if self.audio_out_buf.len() >= 1024 {
+                    self.flush().await;
+                }
+            }
+
+            if self.playing_samples.is_empty() {
+                break;
+            }
+        }
+    }
+
+    async fn submit_buffer(&mut self) {
+        self.audio_sink.sink.send(std::mem::take(&mut self.audio_out_buf)).await.unwrap();
+    }
+
+    async fn flush(&mut self) {
+        if !self.audio_out_buf.is_empty() {
+            self.submit_buffer().await;
+        }
+    }
+
+    async fn play_sample(&mut self, path: &Path) {
+        log::debug!("Queuing sound playback for {:?}", path);
+        let sfx_fd = std::fs::File::open(path).unwrap();
+        let mss = MediaSourceStream::new(Box::new(sfx_fd), Default::default());
+        let meta_opts = Default::default();
+        let fmt_opts = Default::default();
+        let mut hint = Hint::new();
+        // FIXME: use actual file extension
+        hint.with_extension(".mp3");
+        let format = symphonia::default::get_probe()
+            .probe(&hint, mss, fmt_opts, meta_opts)
+            .expect("Unsupported audio format");
+        let track = format.default_track(TrackType::Audio).expect("No audio track");
+        let dec_opts = Default::default();
+        let decoder = symphonia::default::get_codecs()
+            .make_audio_decoder(
+                track.codec_params.as_ref().expect("codec params missing").audio().unwrap(),
+                &dec_opts
+            ).expect("Unsupported audio codec");
+
+        self.playing_samples.push(Sample::new(format, decoder, self.audio_sink.sample_rate));
+    }
+}
+
 pub async fn start_sfx(audio_sink: AudioOutStream) -> SfxControl {
     let (event_sink, mut event_src) = tokio::sync::mpsc::channel(32);
     tokio::spawn(async move {
+        let mut player = Player { audio_sink, audio_out_buf: vec![] , playing_samples: vec![]};
         let sfx_dir = std::path::Path::new("./sfx");
 
         loop {
-            while let Some(event) = event_src.recv().await {
+            tokio::select! {
-                match event {
+                _ = player.process(), if !player.playing_samples.is_empty() => {},
-                    SfxRequest::RandomAmbient => {
+                Some(event) = event_src.recv() => {
-                        let avail_files = std::fs::read_dir(sfx_dir).unwrap();
+                    match event {
-                        let chosen_file = avail_files.choose(&mut rand::rng()).unwrap().unwrap();
+                        SfxRequest::RandomAmbient => {
-                        log::debug!("Queuing ambient sound playback with {:?}", chosen_file);
+                            log::debug!("Playing random audio sample");
-                        let sfx_fd = std::fs::File::open(chosen_file.path()).unwrap();
+                            let avail_files = std::fs::read_dir(sfx_dir.join("ambient")).unwrap();
-                        let mss = MediaSourceStream::new(Box::new(sfx_fd), Default::default());
+                            let chosen_file = avail_files.choose(&mut rand::rng()).unwrap().unwrap();
-                        let meta_opts = Default::default();
+                            player.play_sample(&chosen_file.path()).await;
-                        let fmt_opts = Default::default();
-                        let mut hint = Hint::new();
-                        hint.with_extension(".mp3");
-                        let mut format = symphonia::default::get_probe()
-                            .probe(&hint, mss, fmt_opts, meta_opts)
-                            .expect("Unsupported audio format");
-                        let track = format.default_track(TrackType::Audio).expect("No audio track");
-                        let track_id = track.id;
-                        let dec_opts = Default::default();
-                        let mut decoder = symphonia::default::get_codecs()
-                            .make_audio_decoder(
-                                track.codec_params.as_ref().expect("codec params missing").audio().unwrap(),
-                                &dec_opts
-                            ).expect("Unsupported audio codec");
-
-                        let sample_rate = decoder.codec_params().sample_rate.unwrap();
-                        let channel_num = decoder.codec_params().channels.as_ref().unwrap().count();
-                        log::debug!("Resampling {} -> {}", sample_rate, audio_sink.sample_rate);
-                        // Our resampler works on a mono input
-                        let mut bitrate_resample = resampler::ResamplerFir::new_from_hz(channel_num, sample_rate, audio_sink.sample_rate, Default::default(), Default::default());
-
-                        log::debug!("Starting stream");
-                        let mut audio_out_buf = vec![];
-                        let mut channel_bufs: Vec<f32> = vec![];
-                        loop {
-                            let packet = match format.next_packet() {
-                                Ok(Some(packet)) => packet,
-                                Ok(None) => break,
-                                Err(err) => panic!()
-                            };
-
-                            if packet.track_id != track_id {
-                                continue
-                            }
-
-                            match decoder.decode_ref(&packet.as_packet_ref()) {
-                                Ok(samples) => {
-                                    channel_bufs.resize(samples.samples_interleaved(), 0.);
-                                    samples.copy_to_slice_interleaved(&mut channel_bufs);
-
-                                    let mut resampled = [0.; 2048];
-                                    let (_, write_count) = bitrate_resample.resample(&channel_bufs, &mut resampled).unwrap();
-
-                                    // First we convert the audio feed from stereo down to mono by simple average
-                                    // TODO: This should be something smarter, like a saturating add..?
-                                    let mono_stream = resampled[..write_count].chunks(channel_num).map(|channels| {
-                                        let total_volume = channels.iter().cloned().reduce(|a, b| a + b).unwrap_or_default();
-                                        total_volume / (channel_num as f32)
-                                    });
-
-                                    // Then we write out the resampled audio to our staging buffer
-                                    audio_out_buf.extend(mono_stream);
-
-                                    // Once we have 1024 samples (jack default, I guess), we send it to the audio output
-                                    if audio_out_buf.len() >= 1024 {
-                                        audio_sink.sink.send(audio_out_buf).await.unwrap();
-                                        audio_out_buf = vec![];
-                                    }
-                                },
-                                Err(err) => panic!()
-                            }
                         }
-                        if !audio_out_buf.is_empty() {
-                            audio_sink.sink.send(audio_out_buf).await.unwrap();
-                        }
-                        log::debug!("Playback complete");
                     }
                 }
             }

src/transcription.rs

@@ -91,12 +91,15 @@ pub async fn start_transcription(mut mic_src: AudioInStream) -> TranscriptionCon
                                 let c = client.clone();
                                 let t = transcription_in.clone();
                                 tokio::spawn(async move {
-                                    let response = c.audio().transcription().create(CreateTranscriptionRequest {
+                                    if let Ok(response) = c.audio().transcription().create(CreateTranscriptionRequest {
                                         file: AudioInput { source: InputSource::Bytes { filename: "transcription.wav".into(), bytes } },
                                         model: "gpt-4o-mini-transcribe".into(),
                                         ..Default::default()
-                                    }).await.unwrap();
+                                    }).await {
-                                    t.send(response.text).await.unwrap();
+                                        t.send(response.text).await.unwrap();
+                                    } else {
+                                        log::error!("Error with transcription");
+                                    }
                                 });
                             }
                         }

src/ui.rs

@@ -6,7 +6,7 @@ use tokio::time::Instant;
 use tui_input::{Input, backend::crossterm::EventHandler};
 use tui_skeleton::{AnimationMode, Block, Constraint, SkeletonText};
 
-use crate::{audio::AudioInputControl, prediction::{GeneratedResponses, PredictionAction, SessionControl, SessionUpdate}, scene::{Scene, conversation::{ConversationEntry, Speaker}}, transcription::TranscriptionControl, tts::TtsControl};
+use crate::{audio::AudioInputControl, prediction::{GeneratedResponses, PredictionAction, SessionControl, SessionUpdate}, scene::{Scene, conversation::{ConversationEntry, Speaker}}, sfx::SfxControl, transcription::TranscriptionControl, tts::TtsControl};
 use crate::widgets::*;
 
 #[derive(Debug)]
@@ -30,7 +30,9 @@ pub struct Ui {
     audio: AudioInputControl,
     tts: TtsControl,
     predictions: SessionControl,
-    conversation: Vec<ConversationEntry>
+    conversation: Vec<ConversationEntry>,
+    last_spoken_idx: Option<usize>,
+    sfx: SfxControl
 }
 
 #[derive(Debug)]
@@ -40,7 +42,7 @@ enum FocusState {
 }
 
 impl Ui {
-    pub fn new(predictions: SessionControl, audio: AudioInputControl, transcription: TranscriptionControl, tts: TtsControl) -> Self {
+    pub fn new(predictions: SessionControl, audio: AudioInputControl, transcription: TranscriptionControl, tts: TtsControl, sfx: SfxControl) -> Self {
         Self {
             scene: Default::default(),
             reply_state: Default::default(), 
@@ -58,7 +60,9 @@ impl Ui {
             predictions,
             last_tick: Instant::now(),
             conversation: vec![],
-            reply_options: Default::default()
+            reply_options: Default::default(),
+            last_spoken_idx: None,
+            sfx
         }
     }
 
@@ -193,6 +197,9 @@ impl Ui {
             "/computer" => {
                 self.predictions.insert(PredictionAction::ComputerCommand(arg.to_string())).await;
             },
+            "/ambient" => {
+                self.sfx.play_ambient().await.unwrap()
+            },
             _ => {
                 log::error!("Unknown command. Available commands: /episode [number], /narrative [text], /event [text], /computer [text], /timer [minutes]");
             }
@@ -207,6 +214,7 @@ impl Ui {
                     match key.code {
                         KeyCode::Tab => {
                             self.focus_state = FocusState::UserInput;
+                            self.last_spoken_idx = self.conversation_state.selected();
                             self.conversation_state.select(None);
                             self.reply_state.select_first();
                         },
@@ -220,7 +228,9 @@ impl Ui {
                             let row_num = self.conversation_state.selected().unwrap();
                             if let ConversationEntry::Spoken(Speaker::Eva, text) = &self.conversation[self.conversation.len() - 1 - row_num] {
                                 self.tts.speak(text.clone()).await.unwrap();
+                                self.sfx.play_ambient().await.unwrap();
                                 self.focus_state = FocusState::UserInput;
+                                self.last_spoken_idx = self.conversation_state.selected();
                                 self.conversation_state.select(None);
                                 self.reply_state.select_first();
                             }
@@ -232,7 +242,7 @@ impl Ui {
                     match key.code {
                         KeyCode::Tab => {
                             self.focus_state = FocusState::Conversation;
-                            self.conversation_state.select_first();
+                            self.conversation_state.select(self.last_spoken_idx);
                             self.reply_state.select(None);
                         },
                         KeyCode::Char('r') if key.modifiers.contains(KeyModifiers::CONTROL) => self.predictions.regenerate_options().await,

src/widgets.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use chrono::Duration;
+use itertools::Itertools;
 use ratatui::{layout::Rect, style::Style, text::{Line, Text}, widgets::*};
 use tui_skeleton::Block;
 use ratatui::prelude::*;
@@ -16,7 +17,7 @@ impl Widget for TaskList<'_> {
         let borders = Block::bordered().border_style(style::Color::Red).title("Computer Tasks");
         let wrap_options_unfinished = textwrap::Options::new(area.width as usize).initial_indent("[ ] ").subsequent_indent("    ");
         let wrap_options_finished = textwrap::Options::new(area.width as usize).initial_indent("[X] ").subsequent_indent("    ");
-        let options: Vec<Text> = self.0.iter().map(|(text, is_finished)| {
+        let options: Vec<Text> = self.0.iter().sorted_by(|a, b| Ord::cmp(a.1, b.1)).map(|(text, is_finished)| {
             let (options, color) = if *is_finished {
                 (wrap_options_finished.clone(), style::Color::DarkGray)
             } else {