split out openai api into its own task, to allow the UI to actually stay responsive during network activities

src/main.rs

@@ -1,15 +1,16 @@
 use std::time::Duration;
 
-use async_openai::{Client, config::OpenAIConfig, types::{chat::{ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessage, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, ChatCompletionTool, ChatCompletionTools, CreateChatCompletionRequest, FunctionObject, ResponseFormat, ResponseFormatJsonSchema}}};
+use async_openai::{Client, config::OpenAIConfig, types::chat::{ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessage, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, ChatCompletionTool, ChatCompletionTools, CreateChatCompletionRequest, CreateChatCompletionResponse, FunctionObject, ResponseFormat, ResponseFormatJsonSchema}};
 use futures_timer::Delay;
 use schemars::{JsonSchema, schema_for};
 use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
-use ratatui::{Frame, layout::{Constraint, Direction, Layout}, style::Modifier, widgets::{Block, BorderType, List, ListDirection, ListItem, ListState, Paragraph, Wrap}};
+use ratatui::{Frame, layout::{Constraint, Direction, Layout}, style::Modifier, widgets::{Block, BorderType, Clear, List, ListDirection, ListItem, ListState, Paragraph, Wrap}};
 use throbber_widgets_tui::{Throbber, ThrobberState};
 use crossterm::{event::{self, EventStream, KeyCode, KeyModifiers}};
+use tokio::sync::watch;
 use tui_input::{Input, backend::crossterm::EventHandler};
 use std::process::Command;
 use futures::{StreamExt, future::FutureExt};
@@ -144,17 +145,30 @@ impl Into<CreateChatCompletionRequest> for Scene {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 struct App {
-    client: Client<OpenAIConfig>,
     scene: Scene,
     next_reply_options: Vec<PossibleResponse>,
     reply_state: ListState,
     user_input: Input,
     throbber_state: ThrobberState,
+    prediction_request_sink: watch::Sender<Scene>,
+    is_requesting: bool
 }
 
 impl App {
+    fn new(prediction_request_sink: watch::Sender<Scene>) -> Self {
+        Self {
+            scene: Scene::default(), 
+            next_reply_options: Vec::new(), 
+            reply_state: ListState::default(), 
+            user_input: Input::default(), 
+            throbber_state: ThrobberState::default(),
+            prediction_request_sink,
+            is_requesting: false
+        }
+    }
+
     fn draw_conversation(&self, frame: &mut Frame, area: Rect) {
         let items: Vec<Line> = self.scene.conversation.iter().rev().map(|entry| {
             match entry {
@@ -190,8 +204,12 @@ impl App {
 
     fn draw_io_throbber(&mut self, frame: &mut Frame, area: Rect) {
         let throb_area = area.centered(Constraint::Max(1), Constraint::Max(1));
-        let throbber = Throbber::default();
+        if self.is_requesting {
-        frame.render_stateful_widget(throbber, throb_area, &mut self.throbber_state);
+            let throbber = Throbber::default();
+            frame.render_stateful_widget(throbber, throb_area, &mut self.throbber_state);
+        } else {
+            frame.render_widget(Clear::default(), throb_area);
+        }
     }
 
     fn draw_status(&self, frame: &mut Frame, area: Rect) {
@@ -276,7 +294,7 @@ impl App {
     async fn on_event(&mut self, evt: event::Event) {
         if let Some(key) = evt.as_key_press_event() {
             match key.code {
-                KeyCode::Char('r') if key.modifiers.contains(KeyModifiers::CONTROL) => while !self.update_responses().await {},
+                KeyCode::Char('r') if key.modifiers.contains(KeyModifiers::CONTROL) => self.regenerate_responses(),
                 KeyCode::Down => self.reply_state.select_next(),
                 KeyCode::Up => self.reply_state.select_previous(),
                 KeyCode::Tab => {
@@ -287,7 +305,7 @@ impl App {
                     self.insert_reply(&selected.text);
                     self.save();
                     self.speak(&selected.text.as_str());
-                    while !self.update_responses().await {};
+                    self.regenerate_responses();
                 },
                 KeyCode::Enter => {
                     let next_msg = self.user_input.value_and_reset();
@@ -299,7 +317,7 @@ impl App {
                         self.insert_reply(&selected.text);
                         self.save();
                         self.speak(&selected.text.as_str());
-                        while !self.update_responses().await {};
+                        self.regenerate_responses();
                     } else {
                         if next_msg.starts_with("/") {
                             let mut parts = next_msg.splitn(2, " ");
@@ -324,7 +342,7 @@ impl App {
                             self.insert_chat(&next_msg);
                         }
                         self.save();
-                        while !self.update_responses().await {};
+                        self.regenerate_responses();
                     }
                 },
                 _ => {self.user_input.handle_event(&evt);},
@@ -371,8 +389,14 @@ impl App {
         Command::new("spd-say").arg("-y").arg("English (America)+Linda").arg(text).spawn().unwrap().wait().unwrap();
     }
 
-    async fn update_responses(&mut self) -> bool {
+    fn regenerate_responses(&mut self) {
-        let response = self.client.chat().create(self.scene.clone().into()).await.unwrap();
+        self.prediction_request_sink.send(self.scene.clone()).unwrap();
+        self.next_reply_options.clear();
+        self.is_requesting = true;
+    }
+
+    fn on_response(&mut self, response: CreateChatCompletionResponse) {
+        self.is_requesting = false;
         if let Some(calls) = &response.choices[0].message.tool_calls {
             for call in calls {
                 match call {
@@ -395,13 +419,12 @@ impl App {
                     _ => panic!("Unkown tool call type"),
                 }
             }
-            false
+            self.regenerate_responses();
         } else {
             let json_resp: GeneratedResponses = serde_json::from_str(response.choices[0].message.content.as_ref().unwrap().as_str()).unwrap();
 
             self.next_reply_options = json_resp.responses;
             self.reply_state.select_first();
-            true
         }
     }
 }
@@ -411,7 +434,21 @@ async fn main() {
     color_eyre::install().unwrap();
 
     let mut terminal = ratatui::init();
-    let mut app = App::default();
+
+    let (prediction_in, mut prediction_out) = tokio::sync::watch::channel(None);
+    let (prediction_request_in, mut prediction_request_out) = tokio::sync::watch::channel(Scene::default());
+
+    tokio::spawn(async move {
+        let client: Client<OpenAIConfig> = Client::default();
+        loop {
+            prediction_request_out.changed().await.unwrap();
+            let request = prediction_request_out.borrow().clone();
+            let response = client.chat().create(request.into()).await.unwrap();
+            prediction_in.send(Some(response)).unwrap();
+        }
+    });
+
+    let mut app = App::new(prediction_request_in);
     app.load();
 
     let mut events = EventStream::new();
@@ -420,11 +457,14 @@ async fn main() {
         terminal.draw(|frame| { app.draw(frame)}).unwrap();
         app.throbber_state.calc_next();
 
-        let mut delay = Delay::new(Duration::from_millis(60)).fuse();
+        let delay = Delay::new(Duration::from_millis(60)).fuse();
-        let mut event = events.next().fuse();
+        let event = events.next().fuse();
 
-        select! {
+        tokio::select! {
             _ = delay => (),
+            _ = prediction_out.changed() => {
+                app.on_response(prediction_out.borrow().clone().unwrap());
+            },
             maybe_event = event => {
                 match maybe_event {
                     Some(Ok(event)) => {