From 92660a86a089e7e62dbdeb8982a0d33da9faa04c Mon Sep 17 00:00:00 2001 From: Victoria Fischer Date: Fri, 26 Jun 2026 10:07:46 +0200 Subject: [PATCH] character: stick archive context at the end of the messages, to take advantage of token caching --- src/prediction/character.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/prediction/character.rs b/src/prediction/character.rs index e1166f4..aa8ebfd 100644 --- a/src/prediction/character.rs +++ b/src/prediction/character.rs @@ -51,9 +51,9 @@ impl Character { pub async fn regenerate(&mut self, client: &mut Client, context: ChatCompletionRequestMessage, toolbox: &mut T, output: &mut mpsc::UnboundedSender, schema: &Value) -> Result<(usize, Option), CharacterError> { let mut full_conversation = vec![ self.header_message.clone(), - context ]; full_conversation.extend(self.messages.iter().cloned()); + full_conversation.push(context); let tools = toolbox.tools(); log::debug!("Sending request.."); @@ -75,7 +75,10 @@ impl Character { let response = client.chat().create(request).await?; let tokens_used = if let Some(usage) = response.usage { - log::debug!("{} tokens cast into the void", usage.total_tokens); + let cached_count = usage.prompt_tokens_details.map(|x| x.cached_tokens.unwrap_or_default()).unwrap_or_default(); + let prompt_count = usage.prompt_tokens; + let cache_hit_rate = cached_count as f32 / prompt_count as f32; + log::debug!("{} tokens cast into the void (%{:#.02} cache hit)", usage.total_tokens, cache_hit_rate*100.); usage.total_tokens } else { 0