Skip to content

Mask secrets inside carried compaction summaries; fall back when automatic cache breakpoints are refused #308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fylorn merged 1 commit into from
Oct 9, 2026
Merged

Mask secrets inside carried compaction summaries; fall back when automatic cache breakpoints are refused #308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion crates/tw-dialect/src/anthropic/request.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,9 @@ fn cache_ttl(b: &Value) -> Option<CacheTtl> {
///
/// 断点不算内容：上一轮标在更早位置的那个这一轮挪走了，缓存照样命中（Anthropic 文档里多轮
/// 对话就是这么标的）。太短（不到模型的最小缓存长度）的断点上游直接忽略，不报错，所以不估
/// 长度。思考块不能标，标在它前面的那块上
/// 长度。思考块不能标，标在它前面的那块上。
///
/// 不收 `cache_control` 的兼容接口回 400 时，网关去掉这些断点再发一次（[`crate::cache`]）
fn auto_cache(out: &mut Map<String, Value>) {
fn mark(blocks: Option<&mut Value>) {
let Some(Value::Array(blocks)) = blocks else {
Expand Down
122 changes: 118 additions & 4 deletions crates/tw-dialect/src/bedrock/request.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,67 @@ fn caches(model: &str) -> bool {
m.contains("anthropic.claude") || m.contains("amazon.nova") || m.starts_with("arn:")
}

/// 客户端没标断点时，替它标不标：**只给 AWS 列出支持显式提示缓存的 Claude**。
///
/// 依据是 AWS 的「Supported models, Regions, and explicit caching limits」表
/// （<https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html>，2026-10-09
/// 读的）：Claude 4.5 起的每一个（Haiku 4.5、Sonnet 4.5 / 4.6 / 5 / 5.5、Opus 4.5 ~ 4.8 / 5 /
/// 5.5、Fable、Mythos），加上更早的 Claude 3.7 Sonnet 和 Claude 3.5 Sonnet v2（`20241022`）。
/// 表里没有的 —— Claude 3 Haiku / Sonnet / Opus、3.5 Sonnet v1、3.5 Haiku、Sonnet 4、
/// Opus 4 / 4.1 —— 不自动标：不认的模型收到 `cachePoint` 会拒掉整个请求。
///
/// **按 id 里的家族和版本认**，不按完整的 id：跨区域推理配置的前缀（`us.`、`eu.`、
/// `apac.`、`global.`……）、日期和 `-v1:0` 这些尾巴都不影响。表里还没有的新 id 按版本
/// 判：4.5 起的家族都支持，以后的也算。看不出背后是谁的应用推理配置 ARN 不标。
///
/// 客户端自己标的断点不受这条管（见 [`caches`]）：那是客户端的决定
fn caches_automatically(model: &str) -> bool {
let m = model.to_ascii_lowercase();
// ARN 的最后一段才是模型（推理配置 ARN 里写着 `us.anthropic.claude-…`）
let tail = m.rsplit('/').next().unwrap_or(&m);
let Some(at) = tail.find("anthropic.claude-") else {
return false;
};
let parts: Vec<&str> = tail[at + "anthropic.claude-".len()..]
.split(['-', ':'])
.collect();
// 版本号是一两位的数；八位的是日期
let num = |s: Option<&&str>| {
s.filter(|s| (1..=2).contains(&s.len()))
.and_then(|s| s.parse::<u32>().ok())
};
let (family, major, minor, rest) = match parts.first() {
// 新的写法：`claude-sonnet-4-5-20250929-v1:0`、`claude-opus-5`
Some(f) if f.chars().all(|c| c.is_ascii_alphabetic()) => {
let Some(major) = num(parts.get(1)) else {
return false;
};
match num(parts.get(2)) {
Some(minor) => (*f, major, minor, &parts[3..]),
None => (*f, major, 0, &parts[2..]),
}
}
// 老的写法：`claude-3-7-sonnet-20250219-v1:0`、`claude-3-haiku-20240307-v1:0`
Some(_) => {
let Some(major) = num(parts.first()) else {
return false;
};
let (minor, at) = match num(parts.get(1)) {
Some(minor) => (minor, 2),
None => (0, 1),
};
let Some(family) = parts.get(at) else {
return false;
};
(*family, major, minor, &parts[at + 1..])
}
None => return false,
};
(major, minor) >= (4, 5)
|| (family == "sonnet" && (major, minor) == (3, 7))
|| (family == "sonnet" && (major, minor) == (3, 5) && rest.contains(&"20241022"))
}

/// 客户端没标断点时自动标的，和转给 Anthropic 时同样的四处：工具的末尾、系统提示的末尾、
/// 最后两条用户消息的末尾，各跟一个 5 分钟的 `cachePoint`（Converse 也最多四个）
fn auto_cache(out: &mut Map<String, Value>) {
Expand Down Expand Up @@ -231,10 +292,10 @@ pub fn encode_request(r: &Request, t: &Target, dropped: &mut Dropped) -> Value {
} else if r.tool_choice.is_some() {
dropped.path("tool_choice");
}
// 客户端自己一个断点都没标，模型又是 Claude：替它标（见 `anthropic::request` 的
// `auto_cache`）。Nova 和看不出是谁的推理配置 ARN 不自动标：不认的模型收到
// `cachePoint` 会拒掉整个请求
if r.cache.is_empty() && is_claude(&r.model) {
// 客户端自己一个断点都没标，模型又是 AWS 列出支持缓存的 Claude：替它标（见
// `anthropic::request` 的 `auto_cache`、[`caches_automatically`]）。Nova 和看不出是谁的
// 推理配置 ARN 不自动标：不认的模型收到 `cachePoint` 会拒掉整个请求
if r.cache.is_empty() && caches_automatically(&r.model) {
auto_cache(&mut out);
}

Expand Down Expand Up @@ -1024,6 +1085,59 @@ mod tests {
(v, d.into_vec())
}

#[test]
fn only_the_claude_models_aws_lists_get_automatic_cache_points() {
for (model, yes) in [
// AWS 的表里有的
("anthropic.claude-haiku-5-5", true),
("anthropic.claude-sonnet-5-5", true),
("anthropic.claude-opus-5-5", true),
("anthropic.claude-fable-5-1", true),
("anthropic.claude-mythos-5", true),
("anthropic.claude-opus-5", true),
("anthropic.claude-opus-4-8", true),
("anthropic.claude-opus-4-7", true),
("anthropic.claude-opus-4-6-v1", true),
("anthropic.claude-opus-4-5-20251101-v1:0", true),
("anthropic.claude-sonnet-5", true),
("anthropic.claude-sonnet-4-6", true),
("anthropic.claude-sonnet-4-5-20250929-v1:0", true),
("anthropic.claude-haiku-4-5-20251001-v1:0", true),
("anthropic.claude-3-7-sonnet-20250219-v1:0", true),
("anthropic.claude-3-5-sonnet-20241022-v2:0", true),
// 跨区域推理配置，和指着它的 ARN
("us.anthropic.claude-sonnet-4-5-20250929-v1:0", true),
("global.anthropic.claude-opus-4-7", true),
("apac.anthropic.claude-3-7-sonnet-20250219-v1:0", true),
(
"arn:aws:bedrock:us-east-1:123456789012:inference-profile/eu.anthropic.claude-haiku-4-5-20251001-v1:0",
true,
),
// 表里还没有的新 id：按版本
("anthropic.claude-opus-6", true),
("us.anthropic.claude-sonnet-5-7-20270101-v1:0", true),
// 表里没有的
("anthropic.claude-sonnet-4-20250514-v1:0", false),
("us.anthropic.claude-opus-4-1-20250805-v1:0", false),
("anthropic.claude-opus-4-20250514-v1:0", false),
("anthropic.claude-3-5-sonnet-20240620-v1:0", false),
("anthropic.claude-3-5-haiku-20241022-v1:0", false),
("anthropic.claude-3-haiku-20240307-v1:0", false),
("anthropic.claude-3-opus-20240229-v1:0", false),
("anthropic.claude-v2:1", false),
("anthropic.claude-instant-v1", false),
// 别家的模型、看不出是谁的 ARN
("amazon.nova-pro-v1:0", false),
("meta.llama3-70b-instruct-v1:0", false),
(
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/a1b2c3",
false,
),
] {
assert_eq!(caches_automatically(model), yes, "{model}");
}
}

#[test]
fn cache_points_follow_the_blocks_the_client_marked() {
let r = claude_code_like("us.anthropic.claude-sonnet-4-5-20250929-v1:0");
Expand Down
132 changes: 132 additions & 0 deletions crates/tw-dialect/src/cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
//! 提示缓存的断点，在编码好的请求体上。
//!
//! 转给 Claude 时客户端没标断点的，编码器替它标（`anthropic::request`、`bedrock::request`
//! 里的 `auto_cache`）。有的上游不认 —— 自称 Anthropic 格式的兼容接口、AWS 没列进支持表
//! 的老模型 —— 拒掉整个请求。网关这时去掉断点、同一家再发一次（见 `tw_gateway` 的
//! `cache_marks`），靠的就是这里。
//!
//! **只用在客户端自己一个断点都没标的请求上**：那时请求体里的断点全是编码器加的。客户端
//! 自己标的是它的决定，去掉了它的缓存就白建了。

use serde_json::Value;

use crate::ir::Dialect;

/// 请求体里可能有提示缓存断点吗：按字节找那个键名，不解析。有的不一定真是断点（正文里
/// 提到了这个词），没有的一定不是
pub fn may_have_marks(dialect: Dialect, body: &[u8]) -> bool {
let key: &[u8] = match dialect {
Dialect::Anthropic => b"cache_control",
Dialect::Bedrock => b"cachePoint",
_ => return false,
};
memchr::memmem::find(body, key).is_some()
}

/// 去掉请求体里的提示缓存断点：Anthropic 的 `cache_control`，Converse 的 `cachePoint` 块。
/// 只看工具、系统提示、消息这三处（编码器只标在这里）。
///
/// 没有可去的、不是这两种格式、不是 JSON 的返回 `None`，请求一个字节都不改。
pub fn strip_marks(dialect: Dialect, body: &[u8]) -> Option<Vec<u8>> {
// 绝大多数请求在这里就回去了：一个字都没有，不必解析
if !may_have_marks(dialect, body) {
return None;
}
let mut v: Value = serde_json::from_slice(body).ok()?;
let o = v.as_object_mut()?;
// 一处块列表：Anthropic 的块上去掉 `cache_control`，Converse 去掉 `cachePoint` 块
let strip = |list: Option<&mut Value>| -> bool {
let Some(Value::Array(list)) = list else {
return false;
};
if dialect == Dialect::Anthropic {
let mut changed = false;
for b in list.iter_mut().filter_map(Value::as_object_mut) {
changed |= b.remove("cache_control").is_some();
}
changed
} else {
let before = list.len();
list.retain(|b| b.get("cachePoint").is_none());
list.len() != before
}
};
let mut changed = strip(o.get_mut("system"));
changed |= strip(if dialect == Dialect::Anthropic {
o.get_mut("tools")
} else {
o.get_mut("toolConfig").and_then(|c| c.get_mut("tools"))
});
if let Some(Value::Array(messages)) = o.get_mut("messages") {
for m in messages.iter_mut() {
changed |= strip(m.get_mut("content"));
}
}
changed.then(|| v.to_string().into_bytes())
}

#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;

#[test]
fn the_breakpoints_and_only_them_are_taken_out() {
let mark = json!({"type": "ephemeral"});
let anthropic = json!({
"model": "m",
"tools": [{"name": "f", "input_schema": {}, "cache_control": mark}],
"system": [{"type": "text", "text": "s", "cache_control": mark}],
"messages": [
{"role": "user", "content": [{"type": "text", "text": "mentions cache_control", "cache_control": mark}]},
{"role": "assistant", "content": "plain"}
]
});
let out: Value = serde_json::from_slice(
&strip_marks(Dialect::Anthropic, anthropic.to_string().as_bytes()).unwrap(),
)
.unwrap();
assert_eq!(
out,
json!({
"model": "m",
"tools": [{"name": "f", "input_schema": {}}],
"system": [{"type": "text", "text": "s"}],
"messages": [
{"role": "user", "content": [{"type": "text", "text": "mentions cache_control"}]},
{"role": "assistant", "content": "plain"}
]
})
);

let point = json!({"cachePoint": {"type": "default"}});
let converse = json!({
"system": [{"text": "s"}, point],
"toolConfig": {"tools": [{"toolSpec": {"name": "f"}}, point]},
"messages": [{"role": "user", "content": [{"text": "hi"}, point]}]
});
let out: Value = serde_json::from_slice(
&strip_marks(Dialect::Bedrock, converse.to_string().as_bytes()).unwrap(),
)
.unwrap();
assert_eq!(
out,
json!({
"system": [{"text": "s"}],
"toolConfig": {"tools": [{"toolSpec": {"name": "f"}}]},
"messages": [{"role": "user", "content": [{"text": "hi"}]}]
})
);
}

#[test]
fn nothing_to_take_out_leaves_the_body_alone() {
// 只是在正文里提到了这个词
let body = json!({"messages": [{"role": "user", "content": "what is cache_control?"}]})
.to_string();
assert_eq!(strip_marks(Dialect::Anthropic, body.as_bytes()), None);
assert_eq!(strip_marks(Dialect::Anthropic, b"{}"), None);
assert_eq!(strip_marks(Dialect::Chat, b"{\"cache_control\":1}"), None);
assert_eq!(strip_marks(Dialect::Bedrock, b"not json cachePoint"), None);
}
}
5 changes: 3 additions & 2 deletions crates/tw-dialect/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@
//! 同样两边都用的还有：从响应里旁路嗅出用量（[`usage`]，换算和转换共用各家的
//! `usage()`），拼上游地址（[`url`]），去掉 DeepSeek Harness 只发给 DeepSeek 的
//! 扩展（[`harness`]），在原文上找调用方的正文（[`caller`]，内容过滤读它、删它），
//! 读写各格式里名字不同的请求参数（[`params`]），以及 Codex 的远程压缩转给别家时怎么做
//! （[`compaction`]）。
//! 读写各格式里名字不同的请求参数（[`params`]），Codex 的远程压缩转给别家时怎么做
//! （[`compaction`]），以及请求体上的提示缓存断点（[`cache`]）。

pub mod anthropic;
pub mod bedrock;
pub mod cache;
pub mod caller;
pub mod chat;
pub mod compaction;
Expand Down
Loading
Loading