character: stick archive context at the end of the messages, to take advantage of token caching

src/prediction/character.rs

@@ -51,9 +51,9 @@ impl Character {
     pub async fn regenerate<T: Toolbox>(&mut self, client: &mut Client<OpenAIConfig>, context: ChatCompletionRequestMessage, toolbox: &mut T, output: &mut mpsc::UnboundedSender<PredictionAction>, schema: &Value) -> Result<(usize, Option<Value>), CharacterError> {
         let mut full_conversation = vec![
             self.header_message.clone(),
-            context
         ];
         full_conversation.extend(self.messages.iter().cloned());
+        full_conversation.push(context);
 
         let tools = toolbox.tools();
         log::debug!("Sending request..");
@@ -75,7 +75,10 @@ impl Character {
         let response = client.chat().create(request).await?;
 
         let tokens_used = if let Some(usage) = response.usage {
-            log::debug!("{} tokens cast into the void", usage.total_tokens);
+            let cached_count = usage.prompt_tokens_details.map(|x| x.cached_tokens.unwrap_or_default()).unwrap_or_default();
+            let prompt_count = usage.prompt_tokens;
+            let cache_hit_rate = cached_count as f32 / prompt_count as f32;
+            log::debug!("{} tokens cast into the void (%{:#.02} cache hit)", usage.total_tokens, cache_hit_rate*100.);
             usage.total_tokens
         } else {
             0