From 03ea3728e7ffb6af32310c1ed5ca54f4b4dd8838 Mon Sep 17 00:00:00 2001 From: CJACK Date: Sun, 10 May 2026 17:01:22 +0800 Subject: [PATCH 1/4] fix security advisory issues --- internal/server/router.go | 40 +++++++++++++++++++++++++ internal/server/router_log_test.go | 47 ++++++++++++++++++++++++++++++ internal/webui/handler.go | 7 +++-- internal/webui/handler_test.go | 27 +++++++++++++++++ 4 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 internal/server/router_log_test.go diff --git a/internal/server/router.go b/internal/server/router.go index 07b5e15..b8a78b4 100644 --- a/internal/server/router.go +++ b/internal/server/router.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "net/http" + "net/url" "os" "runtime" "strings" @@ -160,6 +161,16 @@ func (f *filteredLogFormatter) NewLogEntry(r *http.Request) middleware.LogEntry return noopLogEntry{} } } + if r != nil && r.URL != nil { + if redacted, changed := redactSensitiveQueryParams(r.URL); changed { + cloned := *r + clonedURL := *r.URL + clonedURL.RawQuery = redacted + cloned.URL = &clonedURL + cloned.RequestURI = clonedURL.RequestURI() + return f.base.NewLogEntry(&cloned) + } + } return f.base.NewLogEntry(r) } @@ -169,6 +180,35 @@ func (noopLogEntry) Write(_ int, _ int, _ http.Header, _ time.Duration, _ interf func (noopLogEntry) Panic(_ interface{}, _ []byte) {} +func redactSensitiveQueryParams(u *url.URL) (string, bool) { + if u == nil || u.RawQuery == "" { + return "", false + } + values, err := url.ParseQuery(u.RawQuery) + if err != nil && len(values) == 0 { + return "", false + } + changed := false + for name, vals := range values { + if !isSensitiveQueryParam(name) { + continue + } + for i := range vals { + vals[i] = "REDACTED" + } + values[name] = vals + changed = true + } + if !changed { + return "", false + } + return values.Encode(), true +} + +func isSensitiveQueryParam(name string) bool { + return strings.EqualFold(name, "key") || strings.EqualFold(name, "api_key") +} + var defaultCORSAllowHeaders = []string{ "Content-Type", "Authorization", diff --git a/internal/server/router_log_test.go b/internal/server/router_log_test.go new file mode 100644 index 0000000..e5f43df --- /dev/null +++ b/internal/server/router_log_test.go @@ -0,0 +1,47 @@ +package server + +import ( + "bytes" + "log" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5/middleware" +) + +func TestFilteredLogFormatterRedactsSensitiveQueryParams(t *testing.T) { + var buf bytes.Buffer + formatter := &filteredLogFormatter{ + base: &middleware.DefaultLogFormatter{ + Logger: log.New(&buf, "", 0), + NoColor: true, + }, + } + req := httptest.NewRequest( + http.MethodPost, + "/v1beta/models/gemini-2.5-pro:generateContent?key=caller-secret&api_key=second-secret&alt=sse", + nil, + ) + + entry := formatter.NewLogEntry(req) + entry.Write(http.StatusOK, 0, http.Header{}, time.Millisecond, nil) + + got := buf.String() + for _, secret := range []string{"caller-secret", "second-secret"} { + if strings.Contains(got, secret) { + t.Fatalf("log line contains sensitive query value %q: %s", secret, got) + } + } + if !strings.Contains(got, "key=REDACTED") || !strings.Contains(got, "api_key=REDACTED") { + t.Fatalf("log line did not include redacted sensitive params: %s", got) + } + if !strings.Contains(got, "alt=sse") { + t.Fatalf("log line did not preserve non-sensitive query param: %s", got) + } + if req.URL.RawQuery != "key=caller-secret&api_key=second-secret&alt=sse" { + t.Fatalf("request was mutated, RawQuery = %q", req.URL.RawQuery) + } +} diff --git a/internal/webui/handler.go b/internal/webui/handler.go index da9649d..5742bb4 100644 --- a/internal/webui/handler.go +++ b/internal/webui/handler.go @@ -95,11 +95,12 @@ func setStaticContentType(w http.ResponseWriter, fullPath string) { } func (h *Handler) serveFromDisk(w http.ResponseWriter, r *http.Request, staticDir string) { + root := filepath.Clean(staticDir) path := strings.TrimPrefix(r.URL.Path, "/admin") path = strings.TrimPrefix(path, "/") if path != "" && strings.Contains(path, ".") { - full := filepath.Join(staticDir, filepath.Clean(path)) - if !strings.HasPrefix(full, staticDir) { + full := filepath.Join(root, filepath.Clean(path)) + if full != root && !strings.HasPrefix(full, root+string(os.PathSeparator)) { http.NotFound(w, r) return } @@ -116,7 +117,7 @@ func (h *Handler) serveFromDisk(w http.ResponseWriter, r *http.Request, staticDi http.NotFound(w, r) return } - index := filepath.Join(staticDir, "index.html") + index := filepath.Join(root, "index.html") if _, err := os.Stat(index); err != nil { http.Error(w, "index.html not found", http.StatusNotFound) return diff --git a/internal/webui/handler_test.go b/internal/webui/handler_test.go index 99832c5..a819d8c 100644 --- a/internal/webui/handler_test.go +++ b/internal/webui/handler_test.go @@ -78,6 +78,33 @@ func TestServeFromDiskPinsContentType(t *testing.T) { } } +func TestServeFromDiskRejectsSiblingDirectoryWithSharedPrefix(t *testing.T) { + parent := t.TempDir() + staticDir := filepath.Join(parent, "admin") + siblingDir := filepath.Join(parent, "admin-leak") + if err := os.MkdirAll(staticDir, 0o755); err != nil { + t.Fatalf("mkdir static dir: %v", err) + } + if err := os.MkdirAll(siblingDir, 0o755); err != nil { + t.Fatalf("mkdir sibling dir: %v", err) + } + if err := os.WriteFile(filepath.Join(siblingDir, "secret.txt"), []byte("secret"), 0o644); err != nil { + t.Fatalf("write sibling secret: %v", err) + } + + h := &Handler{StaticDir: staticDir} + req := httptest.NewRequest(http.MethodGet, "/admin/../admin-leak/secret.txt", nil) + rec := httptest.NewRecorder() + h.serveFromDisk(rec, req, staticDir) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", rec.Code) + } + if body := rec.Body.String(); strings.Contains(body, "secret") { + t.Fatal("served content from sibling directory") + } +} + // TestSetStaticContentTypeUnknownExtensionFallsThrough verifies that unknown // extensions leave the Content-Type header unset, so http.ServeFile can apply // its own detection (sniffing or mime.TypeByExtension) for cases the pinned From 243860f6d3ee1a483777fa9abe9e5a3937b2b2ad Mon Sep 17 00:00:00 2001 From: CJACK Date: Sun, 10 May 2026 17:02:40 +0800 Subject: [PATCH 2/4] bump version to 4.6.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6016e8a..8ac28bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.6.0 +4.6.1 From e393110121280ba8806376a58824e013dc735cd7 Mon Sep 17 00:00:00 2001 From: CJACK Date: Sun, 10 May 2026 18:00:04 +0800 Subject: [PATCH 3/4] fix toolcall inline code and query redaction --- API.md | 2 +- README.MD | 2 +- docs/prompt-compatibility.md | 2 +- docs/toolcall-semantics.md | 5 +- .../stream-tool-sieve/parse_payload.js | 180 ++++++++++++++++-- .../js/helpers/stream-tool-sieve/sieve.js | 5 +- .../js/helpers/stream-tool-sieve/state.js | 66 +++++++ internal/server/router.go | 55 +++++- internal/server/router_log_test.go | 57 ++++++ internal/toolcall/fence_edge_test.go | 41 ++++ internal/toolcall/toolcalls_candidates.go | 18 +- internal/toolcall/toolcalls_dsml.go | 5 + internal/toolcall/toolcalls_parse.go | 23 +++ internal/toolcall/toolcalls_parse_markup.go | 91 +++++++-- internal/toolcall/toolcalls_scan.go | 12 ++ internal/toolstream/fence_edge_sieve_test.go | 65 +++++++ internal/toolstream/tool_sieve_core.go | 4 +- internal/toolstream/tool_sieve_state.go | 64 +++++++ tests/node/stream-tool-sieve.test.js | 60 ++++++ 19 files changed, 708 insertions(+), 49 deletions(-) diff --git a/API.md b/API.md index 63b4539..3ddd83c 100644 --- a/API.md +++ b/API.md @@ -360,7 +360,7 @@ data: [DONE] - 解析器当前把推荐半角管道符 DSML 外壳（`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`）、DSML wrapper 别名（``、`<|tool_calls>`）、常见 DSML 分隔符漏写形态（如 `<|DSML tool_calls>` / `<|DSML invoke>` / `<|DSML parameter>`）、`DSML` 与工具标签名黏连的常见 typo（如 `` / `` / ``）、控制分隔符漂移（如 `` / 原始 STX `\x02`）、CJK 尖括号、全角感叹号、顿号、PascalCase 本地名、弯引号属性值与属性尾部分隔符漂移（如 `...〈/DSM|parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>` / `` / ``）、任意协议前缀壳（如 ``）和旧式 canonical XML 工具块（`` / `` / ``）作为可执行调用解析；这些非结构性分隔符壳会先归一化回 XML，内部仍以 XML 解析语义为准，CDATA 开头也会容错 `<！[CDATA[` / `<、[CDATA[`。旧式 ``、``、``、``、``、`tool_use`、antml 风格与纯 JSON `tool_calls` 片段默认都会按普通文本处理；完整但 malformed 的 wrapper 同样会作为普通文本释放。 - 解析层不会因为参数值为空而丢弃工具调用；显式空字符串或纯空白参数会按空字符串进入结构化 `tool_calls`。Prompt 会要求模型不要主动输出空参数，缺参/空命令的拒绝应由工具执行侧或客户端 schema 校验负责。 - 当最终可见正文为空但思维链里包含可执行工具调用时，Chat / Responses 会在收尾阶段补发标准 OpenAI `tool_calls` / `function_call` 输出；如果客户端未开启 thinking / reasoning，该思维链只用于检测，不会作为可见正文或 `reasoning_content` 暴露。 -- Markdown fenced code block（例如 ```json ... ```）中的 `tool_calls` 仅视为示例文本，不会被执行。 +- Markdown fenced code block（例如 ```json ... ```）和行内 code span（例如 `` `...` ``）中的 `tool_calls` 仅视为示例文本，不会被执行。 --- diff --git a/README.MD b/README.MD index c32c09c..920201f 100644 --- a/README.MD +++ b/README.MD @@ -372,7 +372,7 @@ Gemini 路由还可以使用 `x-goog-api-key`，或在没有认证头时使用 ` 当请求中带 `tools` 时，DS2API 会做防泄漏处理与结构化转译： -1. 只在**非代码块上下文**启用执行型 toolcall 识别（代码块示例默认不触发） +1. 只在**非 Markdown 代码上下文**启用执行型 toolcall 识别（fenced code block 和行内 code span 中的示例默认不触发） 2. 解析层当前把半角管道符 DSML 外壳视为推荐可执行调用：`<|DSML|tool_calls>` → `<|DSML|invoke name="...">` → `<|DSML|parameter name="...">`；兼容旧式 canonical XML `` → `` → ``，以及若干 DSML 前缀/分隔符漂移。DSML 只是外壳别名，内部仍以 XML 解析语义为准；旧式 `` / `` / `` / ``、``、`tool_use` / antml 变体与纯 JSON `tool_calls` 片段都会按普通文本处理，完整但 malformed 的 wrapper 也会作为普通文本释放 3. `responses` 流式严格使用官方 item 生命周期事件（`response.output_item.*`、`response.content_part.*`、`response.function_call_arguments.*`） 4. `responses` 支持并执行 `tool_choice`（`auto`/`none`/`required`/强制函数）；`required` 违规时非流式返回 `422`，流式返回 `response.failed` diff --git a/docs/prompt-compatibility.md b/docs/prompt-compatibility.md index 7b49865..1cce051 100644 --- a/docs/prompt-compatibility.md +++ b/docs/prompt-compatibility.md @@ -168,7 +168,7 @@ OpenAI Chat / Responses 在标准化后、current input file 之前，会默认 4. 普通直传请求会把“工具描述 + 格式约束”一起并入 system prompt；如果 `current_input_file` 触发，则工具描述/schema 会单独上传成 `DS2API_TOOLS.txt`，live prompt 和 system tool 格式提示都会明确要求模型把 `DS2API_TOOLS.txt` 当作可调用工具和参数 schema 的权威来源。 工具调用正例现在优先示范半角管道符 DSML 风格：`<|DSML|tool_calls>` → `<|DSML|invoke name="...">` → `<|DSML|parameter name="...">`。 -兼容层仍接受旧式纯 `` wrapper，并会容错若干 DSML 标签变体，包括短横线形式 `` / `` / ``、下划线形式 `` / `` / ``，以及其他前缀分隔形态如 `` / `` / ``；标签壳扫描还会把全角 ASCII 漂移归一化，例如 `<ｄＳＭＬ|tool_calls>` 与全角 `＞` 结束符，也会容错 CJK 尖括号、全角感叹号或顿号分隔符、弯引号属性值、PascalCase 本地名和属性尾部分隔符漂移，例如 `...〈/DSM|parameter〉`、`<！DSML！invoke name=“Bash”>`、`<、DSML、tool_calls>`、``、``。更一般地，Go / Node tag 扫描以固定本地标签名 `tool_calls` / `invoke` / `parameter` 为准，标签名前或标签名后的非结构性协议分隔符都会在解析入口剥离，例如 ``、`` 这类控制符或非 ASCII 分隔符漂移也会归一化回现有 XML 标签后继续走同一套 parser；结构性字符如 `<` / `>` / `/` / `=` / 引号、空白和 ASCII 字母数字不会被当作这类分隔符。进入现有 DSML rewrite / XML parse 之前，Go / Node 还会先对“已经识别成工具标签壳的 candidate span”做一次窄 canonicalization：只折叠 wrapper / `invoke` / `parameter` / `name` / `CDATA` / `DSML` 及其壳层分隔符里的 confusable 字符，清理零宽 / BOM / 控制类干扰，并把引号、空白、dash / underscore 变体等统一回可解析的工具语法。这个阶段不会广义改写普通正文、参数内容、CDATA 里的示例文本或其他非工具 XML。CDATA 开头也使用同一类扫描式容错，`` wrapper，并会容错若干 DSML 标签变体，包括短横线形式 `` / `` / ``、下划线形式 `` / `` / ``，以及其他前缀分隔形态如 `` / `` / ``；标签壳扫描还会把全角 ASCII 漂移归一化，例如 `<ｄＳＭＬ|tool_calls>` 与全角 `＞` 结束符，也会容错 CJK 尖括号、全角感叹号或顿号分隔符、弯引号属性值、PascalCase 本地名和属性尾部分隔符漂移，例如 `...〈/DSM|parameter〉`、`<！DSML！invoke name=“Bash”>`、`<、DSML、tool_calls>`、``、``。更一般地，Go / Node tag 扫描以固定本地标签名 `tool_calls` / `invoke` / `parameter` 为准，标签名前或标签名后的非结构性协议分隔符都会在解析入口剥离，例如 ``、`` 这类控制符或非 ASCII 分隔符漂移也会归一化回现有 XML 标签后继续走同一套 parser；结构性字符如 `<` / `>` / `/` / `=` / 引号、空白和 ASCII 字母数字不会被当作这类分隔符。进入现有 DSML rewrite / XML parse 之前，Go / Node 还会先对“已经识别成工具标签壳的 candidate span”做一次窄 canonicalization：只折叠 wrapper / `invoke` / `parameter` / `name` / `CDATA` / `DSML` 及其壳层分隔符里的 confusable 字符，清理零宽 / BOM / 控制类干扰，并把引号、空白、dash / underscore 变体等统一回可解析的工具语法。这个阶段不会广义改写普通正文、参数内容、Markdown 行内 code span、CDATA 里的示例文本或其他非工具 XML。CDATA 开头也使用同一类扫描式容错，`...` 子节点表示；当某个参数体只包含 item 子节点时，Go / Node 解析器会把它还原成数组，避免 `questions` / `options` 这类 schema 中要求 array 的参数被误解析成 `{ "item": ... }` 对象。除此之外，解析器还会回收一些更松散的列表写法，例如 JSON array 字面量或逗号分隔的 JSON 项序列，只要它们足够明确；但 `` 仍然是首选形态。若模型把完整结构化 XML fragment 误包进 CDATA，兼容层会在保护 `content` / `command` 等原文字段的前提下，尝试把非原文字段中的 CDATA XML fragment 还原成 object / array。不过，如果 CDATA 只是单个平面的 XML/HTML 标签，例如 `urgent` 这种行内标记，兼容层会保留原始字符串，不会强行升成 object / array；只有明显表示结构的 CDATA 片段，例如多兄弟节点、嵌套子节点或 `item` 列表，才会触发结构化恢复。对 `command` / `content` 等长文本参数，CDATA 内部的 Markdown fenced DSML / XML 示例会作为原文保护；示例里的 `]]>` 或 `` 不会截断外层工具调用，解析器会继续等待围栏外真正的参数 / wrapper 结束标签。 Go 侧读取 DeepSeek SSE 时不再依赖 `bufio.Scanner` 的固定 2MiB 单行上限；当写文件类工具把很长的 `content` 放在单个 `data:` 行里返回时，非流式收集、流式解析和 auto-continue 透传都会保留完整行，再进入同一套工具解析与序列化流程。 在 assistant 最终回包阶段，如果某个 tool 参数在声明 schema 中明确是 `string`，兼容层会在把解析后的 `tool_calls` / `function_call` 重新序列化成 OpenAI / Responses / Claude 可见参数前，递归把该路径上的 number / bool / object / array 统一转成字符串；其中 object / array 会压成紧凑 JSON 字符串。这个保护只对 schema 明确声明为 string 的路径生效，不会改写本来就是 `number` / `boolean` / `object` / `array` 的参数。这样可以兼容 DeepSeek 输出了结构化片段、但上游客户端工具 schema 又严格要求字符串参数的场景（例如 `content`、`prompt`、`path`、`taskId` 等）。 diff --git a/docs/toolcall-semantics.md b/docs/toolcall-semantics.md index 7988d5a..fb21b61 100644 --- a/docs/toolcall-semantics.md +++ b/docs/toolcall-semantics.md @@ -62,12 +62,12 @@ - 已识别成功的工具调用不会再次回流到普通文本 - 不符合新格式的块不会执行，并继续按原样文本透传 - 如果一个 confusable / 漂移过的工具壳在 candidate-span canonicalization + repair 后仍能形成有效工具调用，wrapper 后面的 suffix prose 会继续按普通文本输出；如果 canonicalization 后仍不满足 wrapper-confidence 或 XML 语义，整块就作为普通文本释放，不会半吞半漏。 -- fenced code block（反引号 `` ``` `` 和波浪线 `~~~`）中的 XML 示例始终按普通文本处理 +- fenced code block（反引号 `` ``` `` 和波浪线 `~~~`）以及 Markdown inline code span（例如 `` `...` ``）中的 XML 示例始终按普通文本处理 - 支持嵌套围栏（如 4 反引号嵌套 3 反引号）和 CDATA 内围栏保护 - 对 `command` / `content` 等长文本参数，CDATA 内部如果包含 Markdown fenced DSML / XML 示例，即使示例里出现 `]]>` / `` 这类看起来像外层结束标签的片段，也会继续按参数原文保留，直到真正位于围栏外的外层结束标签 - CDATA 开头也按扫描式识别，除了标准 `` 或 Markdown inline code 里的 `<|DSML|tool_calls>`）而后面紧跟真正工具调用时，sieve 会跳过不可解析的 mention 候选并继续匹配后续真实工具块，不会因 mention 导致工具调用丢失，也不会截断 mention 后的正文 +- 当文本中 mention 了某种标签名（如 `` 或 Markdown inline code 里的 `<|DSML|tool_calls>`）而后面紧跟真正工具调用时，sieve 会跳过不可解析的 mention 候选并继续匹配后续真实工具块；行内 code span 中即使出现完整 `...` 示例也不会执行，不会因 mention 导致工具调用丢失，也不会截断 mention 后的正文 - Go 侧 SSE 读取不再使用 `bufio.Scanner` 的固定 token 上限；单个 `data:` 行中包含很长的写文件参数时，非流式收集、流式解析与 auto-continue 透传都应保留完整行，再交给 tool parser 处理 另外，`` 的值如果本身是合法 JSON 字面量，也会按结构化值解析，而不是一律保留为字符串。例如 `123`、`true`、`null`、`[1,2]`、`{"a":1}` 都会还原成对应的 number / boolean / null / array / object。 @@ -111,6 +111,7 @@ go test -v -run 'TestParseToolCalls|TestProcessToolSieve' ./internal/toolcall ./ - 混搭标签（DSML wrapper + canonical inner）归一化后正常解析 - 波浪线围栏 `~~~` 内的示例不执行 - 嵌套围栏（4 反引号嵌套 3 反引号）内的示例不执行 +- Markdown 行内 code span 内的完整工具调用示例不执行 - 文本 mention 标签名后紧跟真正工具调用的场景（含同一 wrapper 变体） - 空参数结构化保留，malformed executable-looking XML wrapper 作为文本释放 - 非兼容内容按普通文本透传 diff --git a/internal/js/helpers/stream-tool-sieve/parse_payload.js b/internal/js/helpers/stream-tool-sieve/parse_payload.js index e9fc02f..380a0f3 100644 --- a/internal/js/helpers/stream-tool-sieve/parse_payload.js +++ b/internal/js/helpers/stream-tool-sieve/parse_payload.js @@ -2,8 +2,6 @@ const CDATA_PATTERN = /^(?:<|〈)(?:!|！)\[CDATA\[([\s\S]*?)]](?:>|＞|〉)$/i; const XML_ATTR_PATTERN = /\b([a-z0-9_:-]+)\s*=\s*("([^"]*)"|'([^']*)')/gi; -const XML_TOOL_CALLS_CLOSE_PATTERN = /[<＜][\/／]tool_calls\s*[>＞]/gi; -const XML_INVOKE_START_PATTERN = /[<＜]invoke\b[^>＞]*\bname\s*[=＝]\s*(?:"([^"]*)"|'([^']*)'|“([^”]*)”|‘([^’]*)’|＂([^＂]*)＂|＇([^＇]*)＇)/i; const TOOL_MARKUP_NAMES = [ { raw: 'tool_calls', canonical: 'tool_calls' }, { raw: 'tool-calls', canonical: 'tool_calls', dsmlOnly: true }, @@ -71,6 +69,66 @@ function stripFencedCodeBlocks(text) { return out.join(''); } +function stripMarkdownCodeSpans(text) { + const raw = toStringSafe(text); + if (!raw) { + return ''; + } + let out = ''; + for (let i = 0; i < raw.length;) { + const skipped = skipXmlIgnoredSection(raw, i); + if (skipped.blocked) { + out += raw.slice(i); + break; + } + if (skipped.advanced) { + out += raw.slice(i, skipped.next); + i = skipped.next; + continue; + } + const spanEnd = markdownCodeSpanEnd(raw, i); + if (spanEnd.ok) { + i = spanEnd.end; + continue; + } + out += raw[i]; + i += 1; + } + return out; +} + +function markdownCodeSpanEnd(text, start) { + const raw = toStringSafe(text); + if (start < 0 || start >= raw.length || raw[start] !== '`') { + return { ok: false, end: start }; + } + const count = countLeadingChars(raw, start, '`'); + if (!count) { + return { ok: false, end: start }; + } + let search = start + count; + while (search < raw.length) { + if (raw[search] !== '`') { + search += 1; + continue; + } + const run = countLeadingChars(raw, search, '`'); + if (run === count) { + return { ok: true, end: search + run }; + } + search += run; + } + return { ok: false, end: start }; +} + +function countLeadingChars(text, start, ch) { + let count = 0; + while (start + count < text.length && text[start + count] === ch) { + count += 1; + } + return count; +} + function parseFenceOpenLine(trimmed) { if (trimmed.length < 3) return null; const ch = trimmed[0]; @@ -136,12 +194,12 @@ function parseMarkupToolCalls(text) { if (!raw) { return []; } - let wrappers = findXmlElementBlocks(raw, 'tool_calls'); + let wrappers = findToolCallElementBlocksOutsideIgnored(raw); if (wrappers.length === 0 && hasRepairableXMLToolCallsWrapper(raw)) { const repaired = repairMissingXMLToolCallsOpeningWrapper(raw); if (repaired !== raw) { raw = repaired; - wrappers = findXmlElementBlocks(raw, 'tool_calls'); + wrappers = findToolCallElementBlocksOutsideIgnored(raw); } } const out = []; @@ -157,6 +215,36 @@ function parseMarkupToolCalls(text) { return out; } +function findToolCallElementBlocksOutsideIgnored(text) { + const raw = toStringSafe(text); + const out = []; + for (let searchFrom = 0; searchFrom < raw.length;) { + const tag = findToolMarkupTagOutsideIgnored(raw, searchFrom); + if (!tag) { + break; + } + if (tag.closing || tag.name !== 'tool_calls') { + searchFrom = tag.end + 1; + continue; + } + const closeTag = findMatchingToolMarkupClose(raw, tag); + if (!closeTag) { + searchFrom = tag.end + 1; + continue; + } + const endDelim = xmlTagEndDelimiterLenEndingAt(raw, tag.end); + const attrsEnd = endDelim > 0 ? tag.end + 1 - endDelim : tag.end + 1; + out.push({ + attrs: raw.slice(tag.nameEnd, attrsEnd), + body: raw.slice(tag.end + 1, closeTag.start), + start: tag.start, + end: closeTag.end + 1, + }); + searchFrom = closeTag.end + 1; + } + return out; +} + function normalizeDSMLToolCallMarkup(text) { const raw = toStringSafe(text); if (!raw) { @@ -196,6 +284,11 @@ function containsToolCallWrapperSyntaxOutsideIgnored(text) { i = skipped.next; continue; } + const spanEnd = markdownCodeSpanEnd(raw, i); + if (spanEnd.ok) { + i = spanEnd.end; + continue; + } const tag = scanToolMarkupTagAt(raw, i); if (tag) { if (tag.name !== 'tool_calls') { @@ -232,6 +325,11 @@ function containsToolMarkupSyntaxOutsideIgnored(text) { i = skipped.next; continue; } + const spanEnd = markdownCodeSpanEnd(raw, i); + if (spanEnd.ok) { + i = spanEnd.end; + continue; + } const tag = scanToolMarkupTagAt(raw, i); if (tag) { if (tag.dsmlLike) { @@ -267,6 +365,12 @@ function replaceDSMLToolMarkupOutsideIgnored(text) { i = skipped.next; continue; } + const spanEnd = markdownCodeSpanEnd(raw, i); + if (spanEnd.ok) { + out += raw.slice(i, spanEnd.end); + i = spanEnd.end; + continue; + } const tag = scanToolMarkupTagAt(raw, i); if (tag) { out += `<${tag.closing ? '/' : ''}${tag.name}${raw.slice(tag.nameEnd, tag.end)}>`; @@ -553,6 +657,11 @@ function findToolMarkupTagOutsideIgnored(text, from) { i = skipped.next; continue; } + const spanEnd = markdownCodeSpanEnd(raw, i); + if (spanEnd.ok) { + i = spanEnd.end; + continue; + } const tag = scanToolMarkupTagAt(raw, i); if (tag) { return tag; @@ -987,6 +1096,12 @@ function canonicalizeToolCallCandidateSpans(text) { i = skipped.next; continue; } + const spanEnd = markdownCodeSpanEnd(raw, i); + if (spanEnd.ok) { + out += raw.slice(i, spanEnd.end); + i = spanEnd.end; + continue; + } const tag = scanToolMarkupTagAt(raw, i); if (!tag) { out += raw[i]; @@ -2249,30 +2364,62 @@ function sanitizeLooseCDATA(text) { function hasRepairableXMLToolCallsWrapper(text) { const raw = toStringSafe(text).trim(); - if (!raw || raw.toLowerCase().includes('${raw.slice(invoke.index, close.index)}${raw.slice(close.index + close[0].length)}`; + const invoke = firstToolMarkupTagByName(raw, 'invoke', false); + const close = lastToolMarkupTagByName(raw, 'tool_calls', true); + if (!invoke || !close || invoke.start >= close.start) { + return raw; + } + return `${raw.slice(0, invoke.start)}${raw.slice(invoke.start, close.start)}${raw.slice(close.end + 1)}`; +} + +function firstToolMarkupTagByName(text, name, closing) { + const raw = toStringSafe(text); + for (let searchFrom = 0; searchFrom < raw.length;) { + const tag = findToolMarkupTagOutsideIgnored(raw, searchFrom); + if (!tag) { + break; + } + if (tag.name === name && tag.closing === closing) { + return tag; + } + searchFrom = tag.end + 1; + } + return null; +} + +function lastToolMarkupTagByName(text, name, closing) { + const raw = toStringSafe(text); + let last = null; + for (let searchFrom = 0; searchFrom < raw.length;) { + const tag = findToolMarkupTagOutsideIgnored(raw, searchFrom); + if (!tag) { + break; + } + if (tag.name === name && tag.closing === closing) { + last = tag; + } + searchFrom = tag.end + 1; + } + return last; } function rawNameForTag(tag) { @@ -2494,6 +2641,7 @@ function isOnlyRawValue(obj) { module.exports = { stripFencedCodeBlocks, + stripMarkdownCodeSpans, parseMarkupToolCalls, normalizeDSMLToolCallMarkup, containsToolMarkupSyntaxOutsideIgnored, diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index 0e2d0aa..18898d8 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -3,6 +3,7 @@ const { resetIncrementalToolState, noteText, insideCodeFenceWithState, + insideMarkdownCodeSpanWithState, } = require('./state'); const { trimWrappingJSONFence } = require('./jsonscan'); const { @@ -161,7 +162,7 @@ function splitSafeContentForToolDetection(state, s) { // Only hold back partial XML tool tags. const xmlIdx = findPartialXMLToolTagStart(text); if (xmlIdx >= 0) { - if (insideCodeFenceWithState(state, text.slice(0, xmlIdx))) { + if (insideCodeFenceWithState(state, text.slice(0, xmlIdx)) || insideMarkdownCodeSpanWithState(state, text.slice(0, xmlIdx))) { return [text, '']; } if (xmlIdx > 0) { @@ -182,7 +183,7 @@ function findToolSegmentStart(state, s) { if (!tag) { return -1; } - if (!insideCodeFenceWithState(state, s.slice(0, tag.start))) { + if (!insideCodeFenceWithState(state, s.slice(0, tag.start)) && !insideMarkdownCodeSpanWithState(state, s.slice(0, tag.start))) { return tag.start; } offset = tag.end + 1; diff --git a/internal/js/helpers/stream-tool-sieve/state.js b/internal/js/helpers/stream-tool-sieve/state.js index f9fb2b5..8282f2e 100644 --- a/internal/js/helpers/stream-tool-sieve/state.js +++ b/internal/js/helpers/stream-tool-sieve/state.js @@ -9,6 +9,7 @@ function createToolSieveState() { codeFencePendingTicks: 0, codeFencePendingTildes: 0, codeFenceLineStart: true, + markdownCodeSpanTicks: 0, pendingToolRaw: '', pendingToolCalls: [], disableDeltas: false, @@ -35,6 +36,7 @@ function noteText(state, text) { if (!state || !hasMeaningfulText(text)) { return; } + updateMarkdownCodeSpanState(state, text); updateCodeFenceState(state, text); } @@ -64,6 +66,68 @@ function insideCodeFenceWithState(state, text) { return simulated.stack.length > 0; } +function insideMarkdownCodeSpanWithState(state, text) { + if (!state) { + return simulateMarkdownCodeSpanTicks(null, 0, text) > 0; + } + const ticks = Number.isInteger(state.markdownCodeSpanTicks) ? state.markdownCodeSpanTicks : 0; + return simulateMarkdownCodeSpanTicks(state, ticks, text) > 0; +} + +function updateMarkdownCodeSpanState(state, text) { + if (!state || !hasMeaningfulText(text)) { + return; + } + const ticks = Number.isInteger(state.markdownCodeSpanTicks) ? state.markdownCodeSpanTicks : 0; + state.markdownCodeSpanTicks = simulateMarkdownCodeSpanTicks(state, ticks, text); +} + +function simulateMarkdownCodeSpanTicks(state, initialTicks, text) { + const raw = typeof text === 'string' ? text : ''; + let ticks = Number.isInteger(initialTicks) ? initialTicks : 0; + for (let i = 0; i < raw.length;) { + if (raw[i] !== '`') { + i += 1; + continue; + } + const run = countBacktickRun(raw, i); + if (ticks === 0) { + if (run >= 3 && atMarkdownFenceLineStart(raw, i)) { + i += run; + continue; + } + if (state && insideCodeFenceWithState(state, raw.slice(0, i))) { + i += run; + continue; + } + ticks = run; + } else if (run === ticks) { + ticks = 0; + } + i += run; + } + return ticks; +} + +function countBacktickRun(text, start) { + let count = 0; + while (start + count < text.length && text[start + count] === '`') { + count += 1; + } + return count; +} + +function atMarkdownFenceLineStart(text, idx) { + for (let i = idx - 1; i >= 0; i -= 1) { + const ch = text[i]; + if (ch === ' ' || ch === '\t') { + continue; + } + return ch === '\n' || ch === '\r'; + } + return true; +} + function updateCodeFenceState(state, text) { if (!state) { return; @@ -188,7 +252,9 @@ module.exports = { looksLikeToolExampleContext, insideCodeFence, insideCodeFenceWithState, + insideMarkdownCodeSpanWithState, updateCodeFenceState, + updateMarkdownCodeSpanState, hasMeaningfulText, toStringSafe, }; diff --git a/internal/server/router.go b/internal/server/router.go index b8a78b4..ef6afbf 100644 --- a/internal/server/router.go +++ b/internal/server/router.go @@ -185,8 +185,8 @@ func redactSensitiveQueryParams(u *url.URL) (string, bool) { return "", false } values, err := url.ParseQuery(u.RawQuery) - if err != nil && len(values) == 0 { - return "", false + if err != nil { + return redactSensitiveRawQueryParams(u.RawQuery) } changed := false for name, vals := range values { @@ -205,6 +205,57 @@ func redactSensitiveQueryParams(u *url.URL) (string, bool) { return values.Encode(), true } +func redactSensitiveRawQueryParams(rawQuery string) (string, bool) { + if rawQuery == "" { + return "", false + } + var b strings.Builder + b.Grow(len(rawQuery)) + changed := false + start := 0 + for i := 0; i <= len(rawQuery); i++ { + if i < len(rawQuery) && rawQuery[i] != '&' && rawQuery[i] != ';' { + continue + } + segment := rawQuery[start:i] + b.WriteString(redactSensitiveRawQuerySegment(segment, &changed)) + if i < len(rawQuery) { + b.WriteByte(rawQuery[i]) + } + start = i + 1 + } + if !changed { + return "", false + } + return b.String(), true +} + +func redactSensitiveRawQuerySegment(segment string, changed *bool) string { + if segment == "" { + return segment + } + name := segment + valueStart := -1 + if eq := strings.IndexByte(segment, '='); eq >= 0 { + name = segment[:eq] + valueStart = eq + 1 + } + decodedName, err := url.QueryUnescape(name) + if err != nil { + decodedName = name + } + if !isSensitiveQueryParam(decodedName) { + return segment + } + if changed != nil { + *changed = true + } + if valueStart < 0 { + return name + "=REDACTED" + } + return segment[:valueStart] + "REDACTED" +} + func isSensitiveQueryParam(name string) bool { return strings.EqualFold(name, "key") || strings.EqualFold(name, "api_key") } diff --git a/internal/server/router_log_test.go b/internal/server/router_log_test.go index e5f43df..5ecde48 100644 --- a/internal/server/router_log_test.go +++ b/internal/server/router_log_test.go @@ -45,3 +45,60 @@ func TestFilteredLogFormatterRedactsSensitiveQueryParams(t *testing.T) { t.Fatalf("request was mutated, RawQuery = %q", req.URL.RawQuery) } } + +func TestFilteredLogFormatterRedactsSensitiveQueryParamsWhenMalformed(t *testing.T) { + tests := []struct { + name string + target string + secrets []string + redacted []string + preserved []string + }{ + { + name: "semicolon separator", + target: "/v1beta/models/gemini-2.5-pro:generateContent?key=caller-secret;alt=sse", + secrets: []string{"caller-secret"}, + redacted: []string{"key=REDACTED"}, + preserved: []string{"alt=sse"}, + }, + { + name: "bad escape in sensitive value", + target: "/v1beta/models/gemini-2.5-pro:generateContent?api_key=second-secret%ZZ", + secrets: []string{"second-secret"}, + redacted: []string{"api_key=REDACTED"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + formatter := &filteredLogFormatter{ + base: &middleware.DefaultLogFormatter{ + Logger: log.New(&buf, "", 0), + NoColor: true, + }, + } + req := httptest.NewRequest(http.MethodPost, tt.target, nil) + + entry := formatter.NewLogEntry(req) + entry.Write(http.StatusOK, 0, http.Header{}, time.Millisecond, nil) + + got := buf.String() + for _, secret := range tt.secrets { + if strings.Contains(got, secret) { + t.Fatalf("log line contains sensitive query value %q: %s", secret, got) + } + } + for _, want := range tt.redacted { + if !strings.Contains(got, want) { + t.Fatalf("log line missing redacted query %q: %s", want, got) + } + } + for _, want := range tt.preserved { + if !strings.Contains(got, want) { + t.Fatalf("log line missing preserved query %q: %s", want, got) + } + } + }) + } +} diff --git a/internal/toolcall/fence_edge_test.go b/internal/toolcall/fence_edge_test.go index 5faff0b..f09455a 100644 --- a/internal/toolcall/fence_edge_test.go +++ b/internal/toolcall/fence_edge_test.go @@ -64,3 +64,44 @@ func TestStripFencedCodeBlocks_InlineBackticksNotFence(t *testing.T) { t.Fatalf("expected Before/After, got %q", got) } } + +func TestParseToolCalls_IgnoresMarkdownDocumentationExamples(t *testing.T) { + text := "解析器支持多种工具调用格式。\n\n" + + "入口函数 `ParseToolCalls(text, availableToolNames)` 会返回调用列表。\n\n" + + "核心流程会解析 XML 格式的 `` / `` 标记。\n\n" + + "### 标准 XML 结构\n" + + "```xml\n" + + "\n" + + " \n" + + " config.json\n" + + " \n" + + "\n" + + "```\n\n" + + "DSML 风格形如 `...`，也可能提到 `` 包裹。\n" + + got := ParseToolCallsDetailed(text, []string{"read_file"}) + if len(got.Calls) != 0 { + t.Fatalf("markdown documentation examples should not parse as tool calls, got %#v", got.Calls) + } +} + +func TestParseToolCalls_IgnoresInlineMarkdownToolCallExample(t *testing.T) { + text := "示例：`README.md`" + + got := ParseToolCallsDetailed(text, []string{"read_file"}) + if len(got.Calls) != 0 { + t.Fatalf("inline markdown tool example should not parse as tool calls, got %#v", got.Calls) + } +} + +func TestParseToolCalls_PreservesBackticksInsideToolParameters(t *testing.T) { + text := "echo `date`" + + got := ParseToolCallsDetailed(text, []string{"Bash"}) + if len(got.Calls) != 1 { + t.Fatalf("expected one tool call, got %#v", got.Calls) + } + if got.Calls[0].Input["command"] != "echo `date`" { + t.Fatalf("expected command backticks preserved, got %#v", got.Calls[0].Input["command"]) + } +} diff --git a/internal/toolcall/toolcalls_candidates.go b/internal/toolcall/toolcalls_candidates.go index 187d61a..f9b3cbb 100644 --- a/internal/toolcall/toolcalls_candidates.go +++ b/internal/toolcall/toolcalls_candidates.go @@ -28,6 +28,11 @@ func canonicalizeToolCallCandidateSpans(text string) string { i = next continue } + if end, ok := markdownCodeSpanEnd(text, i); ok { + b.WriteString(text[i:end]) + i = end + continue + } tag, ok := scanToolMarkupTagAt(text, i) if !ok { b.WriteByte(text[i]) @@ -619,19 +624,18 @@ func hasRepairableXMLToolCallsWrapper(text string) bool { if strings.TrimSpace(text) == "" { return false } - if strings.Contains(strings.ToLower(text), "= len(text) || text[start] != '`' { + return start, false + } + count := countLeadingFenceChars(text[start:], '`') + if count == 0 { + return start, false + } + search := start + count + for search < len(text) { + if text[search] != '`' { + search++ + continue + } + run := countLeadingFenceChars(text[search:], '`') + if run == count { + return search + run, true + } + search += run + } + return start, false +} + func cdataStartsBeforeFence(line string) bool { cdataIdx := indexToolCDATAOpen(line, 0) if cdataIdx < 0 { diff --git a/internal/toolcall/toolcalls_parse_markup.go b/internal/toolcall/toolcalls_parse_markup.go index 0d222bd..eab642e 100644 --- a/internal/toolcall/toolcalls_parse_markup.go +++ b/internal/toolcall/toolcalls_parse_markup.go @@ -10,16 +10,14 @@ import ( ) var xmlAttrPattern = regexp.MustCompile(`(?is)\b([a-z0-9_:-]+)\s*=\s*("([^"]*)"|'([^']*)')`) -var xmlToolCallsClosePattern = regexp.MustCompile(`(?is)`) -var xmlInvokeStartPattern = regexp.MustCompile(`(?is)]*\bname\s*=\s*("([^"]*)"|'([^']*)')`) var cdataBRSeparatorPattern = regexp.MustCompile(`(?i)`) func parseXMLToolCalls(text string) []ParsedToolCall { - wrappers := findXMLElementBlocks(text, "tool_calls") + wrappers := findToolCallElementBlocksOutsideIgnored(text) if len(wrappers) == 0 { repaired := repairMissingXMLToolCallsOpeningWrapper(text) if repaired != text { - wrappers = findXMLElementBlocks(repaired, "tool_calls") + wrappers = findToolCallElementBlocksOutsideIgnored(repaired) } } if len(wrappers) == 0 { @@ -41,26 +39,89 @@ func parseXMLToolCalls(text string) []ParsedToolCall { return out } +func findToolCallElementBlocksOutsideIgnored(text string) []xmlElementBlock { + if text == "" { + return nil + } + var out []xmlElementBlock + for searchFrom := 0; searchFrom < len(text); { + tag, ok := FindToolMarkupTagOutsideIgnored(text, searchFrom) + if !ok { + break + } + if tag.Closing || tag.Name != "tool_calls" { + searchFrom = tag.End + 1 + continue + } + closeTag, ok := FindMatchingToolMarkupClose(text, tag) + if !ok { + searchFrom = tag.End + 1 + continue + } + attrsEnd := tag.End + 1 + if delimLen := xmlTagEndDelimiterLenEndingAt(text, tag.End); delimLen > 0 { + attrsEnd = tag.End + 1 - delimLen + } + out = append(out, xmlElementBlock{ + Attrs: text[tag.NameEnd:attrsEnd], + Body: text[tag.End+1 : closeTag.Start], + Start: tag.Start, + End: closeTag.End + 1, + }) + searchFrom = closeTag.End + 1 + } + return out +} + func repairMissingXMLToolCallsOpeningWrapper(text string) string { - lower := strings.ToLower(text) - if strings.Contains(lower, "= closeLoc[0] { + closeTag, ok := lastToolMarkupTagByName(text, "tool_calls", true) + if !ok || invokeTag.Start >= closeTag.Start { return text } - return text[:invokeLoc[0]] + "" + text[invokeLoc[0]:closeLoc[0]] + "" + text[closeLoc[1]:] + return text[:invokeTag.Start] + "" + text[invokeTag.Start:closeTag.Start] + "" + text[closeTag.End+1:] +} + +func firstToolMarkupTagByName(text, name string, closing bool) (ToolMarkupTag, bool) { + for searchFrom := 0; searchFrom < len(text); { + tag, ok := FindToolMarkupTagOutsideIgnored(text, searchFrom) + if !ok { + break + } + if tag.Name == name && tag.Closing == closing { + return tag, true + } + searchFrom = tag.End + 1 + } + return ToolMarkupTag{}, false +} + +func lastToolMarkupTagByName(text, name string, closing bool) (ToolMarkupTag, bool) { + var last ToolMarkupTag + found := false + for searchFrom := 0; searchFrom < len(text); { + tag, ok := FindToolMarkupTagOutsideIgnored(text, searchFrom) + if !ok { + break + } + if tag.Name == name && tag.Closing == closing { + last = tag + found = true + } + searchFrom = tag.End + 1 + } + if !found { + return ToolMarkupTag{}, false + } + return last, true } func parseSingleXMLToolCall(block xmlElementBlock) (ParsedToolCall, bool) { diff --git a/internal/toolcall/toolcalls_scan.go b/internal/toolcall/toolcalls_scan.go index f14ca15..363e1ea 100644 --- a/internal/toolcall/toolcalls_scan.go +++ b/internal/toolcall/toolcalls_scan.go @@ -42,6 +42,10 @@ func ContainsToolMarkupSyntaxOutsideIgnored(text string) (hasDSML, hasCanonical i = next continue } + if end, ok := markdownCodeSpanEnd(text, i); ok { + i = end + continue + } if tag, ok := scanToolMarkupTagAt(text, i); ok { if tag.DSMLLike { hasDSML = true @@ -69,6 +73,10 @@ func ContainsToolCallWrapperSyntaxOutsideIgnored(text string) (hasDSML, hasCanon i = next continue } + if end, ok := markdownCodeSpanEnd(text, i); ok { + i = end + continue + } if tag, ok := scanToolMarkupTagAt(text, i); ok { if tag.Name != "tool_calls" { i = tag.End + 1 @@ -100,6 +108,10 @@ func FindToolMarkupTagOutsideIgnored(text string, start int) (ToolMarkupTag, boo i = next continue } + if end, ok := markdownCodeSpanEnd(text, i); ok { + i = end + continue + } if tag, ok := scanToolMarkupTagAt(text, i); ok { return tag, true } diff --git a/internal/toolstream/fence_edge_sieve_test.go b/internal/toolstream/fence_edge_sieve_test.go index d56335f..81e6f94 100644 --- a/internal/toolstream/fence_edge_sieve_test.go +++ b/internal/toolstream/fence_edge_sieve_test.go @@ -57,3 +57,68 @@ func TestProcessToolSieveNestedFourBacktickFenceDoesNotTrigger(t *testing.T) { t.Fatalf("expected 4-backtick fenced example to stay text, got %d tool calls", toolCalls) } } + +func TestProcessToolSieveMarkdownDocumentationExamplesDoNotTrigger(t *testing.T) { + var state State + chunks := []string{ + "解析器支持多种工具调用格式。\n\n", + "入口函数 `ParseToolCalls(text, availableToolNames)` 会返回调用列表。\n\n", + "核心流程会解析 XML 格式的 `` / `` 标记。\n\n", + "### 标准 XML 结构\n", + "```xml\n", + "\n", + " \n", + " config.json\n", + " \n", + "\n", + "```\n\n", + "DSML 风格形如 `...`，也可能提到 `` 包裹。\n", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var textContent strings.Builder + toolCalls := 0 + for _, evt := range events { + textContent.WriteString(evt.Content) + toolCalls += len(evt.ToolCalls) + } + + if toolCalls != 0 { + t.Fatalf("expected markdown documentation examples to stay text, got %d tool calls", toolCalls) + } + if !strings.Contains(textContent.String(), "标准 XML 结构") || !strings.Contains(textContent.String(), "DSML 风格") { + t.Fatalf("expected documentation text preserved, got %q", textContent.String()) + } +} + +func TestProcessToolSieveInlineMarkdownToolCallSplitAcrossChunksDoesNotTrigger(t *testing.T) { + var state State + chunks := []string{ + "示例：`", + "README.md", + "` 完毕。", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var textContent strings.Builder + toolCalls := 0 + for _, evt := range events { + textContent.WriteString(evt.Content) + toolCalls += len(evt.ToolCalls) + } + + if toolCalls != 0 { + t.Fatalf("expected split inline markdown tool example to stay text, got %d tool calls", toolCalls) + } + if !strings.Contains(textContent.String(), "") || !strings.Contains(textContent.String(), "完毕") { + t.Fatalf("expected inline example text preserved, got %q", textContent.String()) + } +} diff --git a/internal/toolstream/tool_sieve_core.go b/internal/toolstream/tool_sieve_core.go index afe7c7f..3be88ef 100644 --- a/internal/toolstream/tool_sieve_core.go +++ b/internal/toolstream/tool_sieve_core.go @@ -155,7 +155,7 @@ func splitSafeContentForToolDetection(state *State, s string) (safe, hold string return "", "" } if xmlIdx := findPartialXMLToolTagStart(s); xmlIdx >= 0 { - if insideCodeFenceWithState(state, s[:xmlIdx]) { + if insideCodeFenceWithState(state, s[:xmlIdx]) || insideMarkdownCodeSpanWithState(state, s[:xmlIdx]) { return s, "" } if xmlIdx > 0 { @@ -177,7 +177,7 @@ func findToolSegmentStart(state *State, s string) int { return -1 } start := includeDuplicateLeadingLessThan(s, tag.Start) - if !insideCodeFenceWithState(state, s[:start]) { + if !insideCodeFenceWithState(state, s[:start]) && !insideMarkdownCodeSpanWithState(state, s[:start]) { return start } offset = tag.End + 1 diff --git a/internal/toolstream/tool_sieve_state.go b/internal/toolstream/tool_sieve_state.go index cdc70d5..e3d407b 100644 --- a/internal/toolstream/tool_sieve_state.go +++ b/internal/toolstream/tool_sieve_state.go @@ -13,6 +13,7 @@ type State struct { codeFencePendingTicks int codeFencePendingTildes int codeFenceNotLineStart bool // inverted: zero-value false means "at line start" + markdownCodeSpanTicks int pendingToolRaw string pendingToolCalls []toolcall.ParsedToolCall disableDeltas bool @@ -50,6 +51,7 @@ func (s *State) noteText(content string) { if !hasMeaningfulText(content) { return } + updateMarkdownCodeSpanState(s, content) updateCodeFenceState(s, content) } @@ -78,6 +80,68 @@ func insideCodeFence(text string) bool { return len(simulateCodeFenceState(nil, 0, 0, true, text).stack) > 0 } +func insideMarkdownCodeSpanWithState(state *State, text string) bool { + if state == nil { + return simulateMarkdownCodeSpanTicks(nil, 0, text) > 0 + } + return simulateMarkdownCodeSpanTicks(state, state.markdownCodeSpanTicks, text) > 0 +} + +func updateMarkdownCodeSpanState(state *State, text string) { + if state == nil || !hasMeaningfulText(text) { + return + } + state.markdownCodeSpanTicks = simulateMarkdownCodeSpanTicks(state, state.markdownCodeSpanTicks, text) +} + +func simulateMarkdownCodeSpanTicks(state *State, initialTicks int, text string) int { + ticks := initialTicks + for i := 0; i < len(text); { + if text[i] != '`' { + i++ + continue + } + run := countBacktickRun(text, i) + if ticks == 0 { + if run >= 3 && atMarkdownFenceLineStart(text, i) { + i += run + continue + } + if state != nil && insideCodeFenceWithState(state, text[:i]) { + i += run + continue + } + ticks = run + } else if run == ticks { + ticks = 0 + } + i += run + } + return ticks +} + +func countBacktickRun(text string, start int) int { + count := 0 + for start+count < len(text) && text[start+count] == '`' { + count++ + } + return count +} + +func atMarkdownFenceLineStart(text string, idx int) bool { + for i := idx - 1; i >= 0; i-- { + switch text[i] { + case ' ', '\t': + continue + case '\n', '\r': + return true + default: + return false + } + } + return true +} + func updateCodeFenceState(state *State, text string) { if state == nil || !hasMeaningfulText(text) { return diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index b989fdb..f23777d 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -568,6 +568,19 @@ test('parseToolCalls skips prose mention of same wrapper variant', () => { assert.equal(calls[0].input.command, 'git status'); }); +test('parseToolCalls ignores inline markdown tool example', () => { + const payload = '示例：`README.md`'; + const calls = parseToolCalls(payload, ['read_file']); + assert.equal(calls.length, 0); +}); + +test('parseToolCalls preserves backticks inside tool parameters', () => { + const payload = 'echo `date`'; + const calls = parseToolCalls(payload, ['Bash']); + assert.equal(calls.length, 1); + assert.equal(calls[0].input.command, 'echo `date`'); +}); + test('sieve emits tool_calls after prose mentions same wrapper variant', () => { const events = runSieve([ 'Summary: support canonical and DSML <|DSML|tool_calls> wrappers.\n\n', @@ -584,6 +597,53 @@ test('sieve emits tool_calls after prose mentions same wrapper variant', () => { assert.equal(collectText(events).includes('Summary:'), true); }); +test('sieve ignores markdown documentation examples', () => { + const events = runSieve([ + '解析器支持多种工具调用格式。\n\n', + '入口函数 `ParseToolCalls(text, availableToolNames)` 会返回调用列表。\n\n', + '核心流程会解析 XML 格式的 `` / `` 标记。\n\n', + '### 标准 XML 结构\n', + '```xml\n', + '\n', + ' \n', + ' config.json\n', + ' \n', + '\n', + '```\n\n', + 'DSML 风格形如 `...`，也可能提到 `` 包裹。\n', + ], ['read_file']); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + const text = collectText(events); + assert.equal(finalCalls.length, 0); + assert.equal(text.includes('标准 XML 结构'), true); + assert.equal(text.includes('DSML 风格'), true); +}); + +test('sieve ignores inline markdown tool example split across chunks', () => { + const events = runSieve([ + '示例：`', + 'README.md', + '` 完毕。', + ], ['read_file']); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + const text = collectText(events); + assert.equal(finalCalls.length, 0); + assert.equal(text.includes(''), true); + assert.equal(text.includes('完毕'), true); +}); + +test('sieve emits real tool after split inline markdown tool example closes', () => { + const events = runSieve([ + '示例：`', + 'README.md', + '` ', + 'real.md', + ], ['read_file']); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 1); + assert.equal(finalCalls[0].input.path, 'real.md'); +}); + test('sieve emits tool_calls for DSML space-separator typo', () => { const events = runSieve([ '准备读取文件。\n', From 77a47ada4e509d2f60730296b39446c2861f0fef Mon Sep 17 00:00:00 2001 From: CJACK Date: Sun, 10 May 2026 18:41:51 +0800 Subject: [PATCH 4/4] Fix tool detection when unclosed backtick precedes tool call Handles cases where a stray backtick opens an inline code span but is never closed. Previously, any subsequent XML tool tag was treated as inside markdown code and ignored. Now, tool tags are detected after an unclosed backtick, and the markdown state is reset when the backtick is confirmed to be literal text at stream boundaries. Co-Authored-By: Claude Opus 4.6 --- .../js/helpers/stream-tool-sieve/sieve.js | 114 +++++++++++++++++- internal/toolstream/fence_edge_sieve_test.go | 55 +++++++++ internal/toolstream/tool_sieve_core.go | 104 +++++++++++++++- internal/toolstream/tool_sieve_state.go | 7 -- tests/node/stream-tool-sieve.test.js | 21 ++++ 5 files changed, 287 insertions(+), 14 deletions(-) diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index 18898d8..961211c 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -3,7 +3,6 @@ const { resetIncrementalToolState, noteText, insideCodeFenceWithState, - insideMarkdownCodeSpanWithState, } = require('./state'); const { trimWrappingJSONFence } = require('./jsonscan'); const { @@ -71,10 +70,17 @@ function processToolSieveChunk(state, chunk, toolNames) { break; } const start = findToolSegmentStart(state, pending); + if (start === HOLD_TOOL_SEGMENT_START) { + break; + } if (start >= 0) { const prefix = pending.slice(0, start); if (prefix) { + const resetMarkdownSpan = shouldResetUnclosedMarkdownPrefix(state, prefix, pending.slice(start)); noteText(state, prefix); + if (resetMarkdownSpan) { + state.markdownCodeSpanTicks = 0; + } events.push({ type: 'text', text: prefix }); } state.pending = ''; @@ -99,6 +105,10 @@ function flushToolSieve(state, toolNames) { return []; } const events = processToolSieveChunk(state, '', toolNames); + if (state.pending && Number.isInteger(state.markdownCodeSpanTicks) && state.markdownCodeSpanTicks > 0) { + state.markdownCodeSpanTicks = 0; + events.push(...processToolSieveChunk(state, '', toolNames)); + } if (Array.isArray(state.pendingToolCalls) && state.pendingToolCalls.length > 0) { events.push({ type: 'tool_calls', calls: state.pendingToolCalls }); state.pendingToolRaw = ''; @@ -162,9 +172,18 @@ function splitSafeContentForToolDetection(state, s) { // Only hold back partial XML tool tags. const xmlIdx = findPartialXMLToolTagStart(text); if (xmlIdx >= 0) { - if (insideCodeFenceWithState(state, text.slice(0, xmlIdx)) || insideMarkdownCodeSpanWithState(state, text.slice(0, xmlIdx))) { + if (insideCodeFenceWithState(state, text.slice(0, xmlIdx))) { return [text, '']; } + const markdown = markdownCodeSpanStateAt(state, text.slice(0, xmlIdx)); + if (markdown.ticks > 0) { + if (markdownCodeSpanCloses(text.slice(xmlIdx), markdown.ticks)) { + return [text, '']; + } + if (markdown.fromPrior) { + return ['', text]; + } + } if (xmlIdx > 0) { return [text.slice(0, xmlIdx), text.slice(xmlIdx)]; } @@ -173,6 +192,8 @@ function splitSafeContentForToolDetection(state, s) { return [text, '']; } +const HOLD_TOOL_SEGMENT_START = -2; + function findToolSegmentStart(state, s) { if (!s) { return -1; @@ -183,13 +204,98 @@ function findToolSegmentStart(state, s) { if (!tag) { return -1; } - if (!insideCodeFenceWithState(state, s.slice(0, tag.start)) && !insideMarkdownCodeSpanWithState(state, s.slice(0, tag.start))) { + if (insideCodeFenceWithState(state, s.slice(0, tag.start))) { + offset = tag.end + 1; + continue; + } + const markdown = markdownCodeSpanStateAt(state, s.slice(0, tag.start)); + if (markdown.ticks === 0) { return tag.start; } - offset = tag.end + 1; + if (markdownCodeSpanCloses(s.slice(tag.start), markdown.ticks)) { + offset = tag.end + 1; + continue; + } + if (markdown.fromPrior) { + return HOLD_TOOL_SEGMENT_START; + } + return tag.start; } } +function markdownCodeSpanStateAt(state, text) { + const raw = typeof text === 'string' ? text : ''; + let ticks = state && Number.isInteger(state.markdownCodeSpanTicks) ? state.markdownCodeSpanTicks : 0; + let fromPrior = ticks > 0; + for (let i = 0; i < raw.length;) { + if (raw[i] !== '`') { + i += 1; + continue; + } + const run = countBacktickRun(raw, i); + if (ticks === 0) { + if (run >= 3 && atMarkdownFenceLineStart(raw, i)) { + i += run; + continue; + } + if (state && insideCodeFenceWithState(state, raw.slice(0, i))) { + i += run; + continue; + } + ticks = run; + fromPrior = false; + } else if (run === ticks) { + ticks = 0; + fromPrior = false; + } + i += run; + } + return { ticks, fromPrior }; +} + +function markdownCodeSpanCloses(text, ticks) { + const raw = typeof text === 'string' ? text : ''; + if (!Number.isInteger(ticks) || ticks <= 0) { + return false; + } + for (let i = 0; i < raw.length;) { + if (raw[i] !== '`') { + i += 1; + continue; + } + const run = countBacktickRun(raw, i); + if (run === ticks) { + return true; + } + i += run; + } + return false; +} + +function shouldResetUnclosedMarkdownPrefix(state, prefix, suffix) { + const markdown = markdownCodeSpanStateAt(state, prefix); + return markdown.ticks > 0 && !markdown.fromPrior && !markdownCodeSpanCloses(suffix, markdown.ticks); +} + +function countBacktickRun(text, start) { + let count = 0; + while (start + count < text.length && text[start + count] === '`') { + count += 1; + } + return count; +} + +function atMarkdownFenceLineStart(text, idx) { + for (let i = idx - 1; i >= 0; i -= 1) { + const ch = text[i]; + if (ch === ' ' || ch === '\t') { + continue; + } + return ch === '\n' || ch === '\r'; + } + return true; +} + function consumeToolCapture(state, toolNames) { const captured = state.capture || ''; if (!captured) { diff --git a/internal/toolstream/fence_edge_sieve_test.go b/internal/toolstream/fence_edge_sieve_test.go index 81e6f94..035108d 100644 --- a/internal/toolstream/fence_edge_sieve_test.go +++ b/internal/toolstream/fence_edge_sieve_test.go @@ -122,3 +122,58 @@ func TestProcessToolSieveInlineMarkdownToolCallSplitAcrossChunksDoesNotTrigger(t t.Fatalf("expected inline example text preserved, got %q", textContent.String()) } } + +func TestProcessToolSieveUnclosedInlineMarkdownBeforeToolDoesTrigger(t *testing.T) { + var state State + input := "note with stray ` before real call " + + "real.md" + + var events []Event + events = append(events, ProcessChunk(&state, input, []string{"read_file"})...) + events = append(events, Flush(&state, []string{"read_file"})...) + + var textContent strings.Builder + var calls []string + for _, evt := range events { + textContent.WriteString(evt.Content) + for _, call := range evt.ToolCalls { + if path, _ := call.Input["path"].(string); path != "" { + calls = append(calls, path) + } + } + } + + if len(calls) != 1 || calls[0] != "real.md" { + t.Fatalf("expected real tool call after stray backtick, got %#v from events %#v", calls, events) + } + if !strings.Contains(textContent.String(), "stray ` before real call") { + t.Fatalf("expected stray-backtick prefix preserved, got %q", textContent.String()) + } +} + +func TestProcessToolSieveUnclosedInlineMarkdownBeforeSplitToolDoesTriggerOnFlush(t *testing.T) { + var state State + chunks := []string{ + "note with stray ` before real call ", + "real.md", + } + + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var calls []string + for _, evt := range events { + for _, call := range evt.ToolCalls { + if path, _ := call.Input["path"].(string); path != "" { + calls = append(calls, path) + } + } + } + + if len(calls) != 1 || calls[0] != "real.md" { + t.Fatalf("expected split real tool call after stray backtick, got %#v from events %#v", calls, events) + } +} diff --git a/internal/toolstream/tool_sieve_core.go b/internal/toolstream/tool_sieve_core.go index 3be88ef..a0791f4 100644 --- a/internal/toolstream/tool_sieve_core.go +++ b/internal/toolstream/tool_sieve_core.go @@ -57,10 +57,17 @@ func ProcessChunk(state *State, chunk string, toolNames []string) []Event { break } start := findToolSegmentStart(state, pending) + if start == holdToolSegmentStart { + break + } if start >= 0 { prefix := pending[:start] if prefix != "" { + resetMarkdownSpan := shouldResetUnclosedMarkdownPrefix(state, prefix, pending[start:]) state.noteText(prefix) + if resetMarkdownSpan { + state.markdownCodeSpanTicks = 0 + } events = append(events, Event{Content: prefix}) } state.pending.Reset() @@ -88,6 +95,13 @@ func Flush(state *State, toolNames []string) []Event { return nil } events := ProcessChunk(state, "", toolNames) + if state.pending.Len() > 0 && state.markdownCodeSpanTicks > 0 { + // At end of stream, an unmatched backtick is literal Markdown text. + // Re-scan pending content so a real tool call after that stray + // backtick is not permanently hidden by inline-code state. + state.markdownCodeSpanTicks = 0 + events = append(events, ProcessChunk(state, "", toolNames)...) + } if len(state.pendingToolCalls) > 0 { events = append(events, Event{ToolCalls: state.pendingToolCalls}) state.pendingToolRaw = "" @@ -155,9 +169,18 @@ func splitSafeContentForToolDetection(state *State, s string) (safe, hold string return "", "" } if xmlIdx := findPartialXMLToolTagStart(s); xmlIdx >= 0 { - if insideCodeFenceWithState(state, s[:xmlIdx]) || insideMarkdownCodeSpanWithState(state, s[:xmlIdx]) { + if insideCodeFenceWithState(state, s[:xmlIdx]) { return s, "" } + markdown := markdownCodeSpanStateAt(state, s[:xmlIdx]) + if markdown.ticks > 0 { + if markdownCodeSpanCloses(s[xmlIdx:], markdown.ticks) { + return s, "" + } + if markdown.fromPrior { + return "", s + } + } if xmlIdx > 0 { return s[:xmlIdx], s[xmlIdx:] } @@ -166,6 +189,8 @@ func splitSafeContentForToolDetection(state *State, s string) (safe, hold string return s, "" } +const holdToolSegmentStart = -2 + func findToolSegmentStart(state *State, s string) int { if s == "" { return -1 @@ -177,13 +202,86 @@ func findToolSegmentStart(state *State, s string) int { return -1 } start := includeDuplicateLeadingLessThan(s, tag.Start) - if !insideCodeFenceWithState(state, s[:start]) && !insideMarkdownCodeSpanWithState(state, s[:start]) { + if insideCodeFenceWithState(state, s[:start]) { + offset = tag.End + 1 + continue + } + markdown := markdownCodeSpanStateAt(state, s[:start]) + if markdown.ticks == 0 { return start } - offset = tag.End + 1 + if markdownCodeSpanCloses(s[start:], markdown.ticks) { + offset = tag.End + 1 + continue + } + if markdown.fromPrior { + return holdToolSegmentStart + } + return start } } +type markdownCodeSpanScan struct { + ticks int + fromPrior bool +} + +func markdownCodeSpanStateAt(state *State, text string) markdownCodeSpanScan { + ticks := 0 + fromPrior := false + if state != nil && state.markdownCodeSpanTicks > 0 { + ticks = state.markdownCodeSpanTicks + fromPrior = true + } + for i := 0; i < len(text); { + if text[i] != '`' { + i++ + continue + } + run := countBacktickRun(text, i) + if ticks == 0 { + if run >= 3 && atMarkdownFenceLineStart(text, i) { + i += run + continue + } + if state != nil && insideCodeFenceWithState(state, text[:i]) { + i += run + continue + } + ticks = run + fromPrior = false + } else if run == ticks { + ticks = 0 + fromPrior = false + } + i += run + } + return markdownCodeSpanScan{ticks: ticks, fromPrior: fromPrior} +} + +func markdownCodeSpanCloses(text string, ticks int) bool { + if ticks <= 0 { + return false + } + for i := 0; i < len(text); { + if text[i] != '`' { + i++ + continue + } + run := countBacktickRun(text, i) + if run == ticks { + return true + } + i += run + } + return false +} + +func shouldResetUnclosedMarkdownPrefix(state *State, prefix, suffix string) bool { + markdown := markdownCodeSpanStateAt(state, prefix) + return markdown.ticks > 0 && !markdown.fromPrior && !markdownCodeSpanCloses(suffix, markdown.ticks) +} + func includeDuplicateLeadingLessThan(s string, idx int) int { for idx > 0 && s[idx-1] == '<' { idx-- diff --git a/internal/toolstream/tool_sieve_state.go b/internal/toolstream/tool_sieve_state.go index e3d407b..2c1711c 100644 --- a/internal/toolstream/tool_sieve_state.go +++ b/internal/toolstream/tool_sieve_state.go @@ -80,13 +80,6 @@ func insideCodeFence(text string) bool { return len(simulateCodeFenceState(nil, 0, 0, true, text).stack) > 0 } -func insideMarkdownCodeSpanWithState(state *State, text string) bool { - if state == nil { - return simulateMarkdownCodeSpanTicks(nil, 0, text) > 0 - } - return simulateMarkdownCodeSpanTicks(state, state.markdownCodeSpanTicks, text) > 0 -} - func updateMarkdownCodeSpanState(state *State, text string) { if state == nil || !hasMeaningfulText(text) { return diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index f23777d..c5f09d8 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -632,6 +632,27 @@ test('sieve ignores inline markdown tool example split across chunks', () => { assert.equal(text.includes('完毕'), true); }); +test('sieve emits real tool after unclosed inline markdown in same chunk', () => { + const events = runSieve([ + 'note with stray ` before real call real.md', + ], ['read_file']); + const text = collectText(events); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 1); + assert.equal(finalCalls[0].input.path, 'real.md'); + assert.equal(text.includes('stray ` before real call'), true); +}); + +test('sieve emits real tool after unclosed inline markdown across chunks', () => { + const events = runSieve([ + 'note with stray ` before real call ', + 'real.md', + ], ['read_file']); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 1); + assert.equal(finalCalls[0].input.path, 'real.md'); +}); + test('sieve emits real tool after split inline markdown tool example closes', () => { const events = runSieve([ '示例：`',