From a13293e113cd3195e4869be340e2b25c0aa3df40 Mon Sep 17 00:00:00 2001 From: CJACK Date: Mon, 27 Apr 2026 13:39:50 +0800 Subject: [PATCH] feat: expand DSML tool-call alias and fence handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for DSML wrapper aliases (, <|tool_calls>, <｜tool_calls>) alongside canonical XML. Normalize mixed DSML/canonical tags instead of rejecting them. Add tilde fence (~~~) support, fix nested fence and unclosed fence handling, support CDATA-protected fence content, and skip prose mentions when scanning for real tool blocks. Mirror all changes between Go and Node.js runtimes. Co-Authored-By: Claude Opus 4.7 --- API.md | 4 +- docs/toolcall-semantics.md | 11 +- .../js/helpers/stream-tool-sieve/parse.js | 2 +- .../stream-tool-sieve/parse_payload.js | 124 +++- .../js/helpers/stream-tool-sieve/sieve-xml.js | 189 +++++- .../js/helpers/stream-tool-sieve/sieve.js | 4 + .../js/helpers/stream-tool-sieve/state.js | 53 +- .../stream-tool-sieve/tool-keywords.js | 14 + internal/toolcall/fence_edge_test.go | 66 +++ internal/toolcall/toolcalls_dsml.go | 28 +- internal/toolcall/toolcalls_parse.go | 16 +- internal/toolcall/toolcalls_parse_markup.go | 3 +- internal/toolcall/toolcalls_test.go | 33 +- internal/toolstream/complex_edge_test.go | 556 ++++++++++++++++++ internal/toolstream/fence_edge_sieve_test.go | 59 ++ internal/toolstream/tool_sieve_state.go | 111 ++-- internal/toolstream/tool_sieve_xml.go | 199 ++++++- internal/toolstream/tool_sieve_xml_test.go | 93 +++ tests/node/stream-tool-sieve.test.js | 84 ++- 19 files changed, 1524 insertions(+), 125 deletions(-) create mode 100644 internal/toolcall/fence_edge_test.go create mode 100644 internal/toolstream/complex_edge_test.go create mode 100644 internal/toolstream/fence_edge_sieve_test.go diff --git a/API.md b/API.md index 7bf19f3..17a2f1e 100644 --- a/API.md +++ b/API.md @@ -37,7 +37,7 @@ - OpenAI / Claude / Gemini 三套协议已统一挂在同一 `chi` 路由树上，由 `internal/server/router.go` 负责装配。 - 适配器层职责收敛为：**请求归一化 → DeepSeek 调用 → 协议形态渲染**，减少历史版本中“同能力多处实现”的分叉。 -- Tool Calling 的解析策略在 Go 与 Node Runtime 间保持一致：推荐模型输出 DSML 外壳 `<|DSML|tool_calls>` → `<|DSML|invoke name="...">` → `<|DSML|parameter name="...">`；兼容层也接受旧式 canonical XML `` → `` → ``，内部仍以 XML 解析语义为准，并在流式场景执行防泄漏筛分。 +- Tool Calling 的解析策略在 Go 与 Node Runtime 间保持一致：推荐模型输出 DSML 外壳 `<|DSML|tool_calls>` → `<|DSML|invoke name="...">` → `<|DSML|parameter name="...">`；兼容层也接受 DSML wrapper 别名 ``、`<|tool_calls>`、`<｜tool_calls>` 以及旧式 canonical XML `` → `` → ``，内部仍以 XML 解析语义为准，并在流式场景执行防泄漏筛分。 - `Admin API` 将配置与运行时策略分开：`/admin/config*` 管静态配置，`/admin/settings*` 管运行时行为。 --- @@ -344,7 +344,7 @@ data: [DONE] 补充说明： - **非代码块上下文**下，工具负载即使与普通文本混合，也会按特征识别并产出可执行 tool call（前后普通文本仍可透传）。 -- 解析器当前把 DSML 外壳（`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`）和旧式 canonical XML 工具块（`` / `` / ``）作为可执行调用解析；DSML 会先归一化回 XML，内部仍以 XML 解析语义为准。旧式 ``、``、``、``、``、`tool_use`、antml 风格与纯 JSON `tool_calls` 片段默认都会按普通文本处理。 +- 解析器当前把 DSML 外壳（`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`）、DSML wrapper 别名（``、`<|tool_calls>`、`<｜tool_calls>`）和旧式 canonical XML 工具块（`` / `` / ``）作为可执行调用解析；DSML 会先归一化回 XML，内部仍以 XML 解析语义为准。旧式 ``、``、``、``、``、`tool_use`、antml 风格与纯 JSON `tool_calls` 片段默认都会按普通文本处理。 - 当最终可见正文为空但思维链里包含可执行工具调用时，Chat / Responses 会在收尾阶段补发标准 OpenAI `tool_calls` / `function_call` 输出；如果客户端未开启 thinking / reasoning，该思维链只用于检测，不会作为可见正文或 `reasoning_content` 暴露。 - Markdown fenced code block（例如 ```json ... ```）中的 `tool_calls` 仅视为示例文本，不会被执行。 diff --git a/docs/toolcall-semantics.md b/docs/toolcall-semantics.md index c15da11..fe38c72 100644 --- a/docs/toolcall-semantics.md +++ b/docs/toolcall-semantics.md @@ -51,11 +51,13 @@ 在流式链路中（Go / Node 一致）： -- DSML `<|DSML|tool_calls>` wrapper 和 canonical `` wrapper 都会进入结构化捕获 +- DSML `<|DSML|tool_calls>` wrapper 及其兼容变体（``、`<｜tool_calls>`、`<|tool_calls>`）和 canonical `` wrapper 都会进入结构化捕获 - 如果流里直接从 invoke 开始，但后面补上了 closing wrapper，Go 流式筛分也会按缺失 opening wrapper 的修复路径尝试恢复 - 已识别成功的工具调用不会再次回流到普通文本 - 不符合新格式的块不会执行，并继续按原样文本透传 -- fenced code block 中的 XML 示例始终按普通文本处理 +- fenced code block（反引号 `` ``` `` 和波浪线 `~~~`）中的 XML 示例始终按普通文本处理 +- 支持嵌套围栏（如 4 反引号嵌套 3 反引号）和 CDATA 内围栏保护 +- 当文本中 mention 了某种标签名（如 `` 或 Markdown inline code 里的 `<|DSML|tool_calls>`）而后面紧跟真正工具调用时，sieve 会跳过不可解析的 mention 候选并继续匹配后续真实工具块，不会因 mention 导致工具调用丢失，也不会截断 mention 后的正文 ## 4) 输出结构 @@ -85,5 +87,10 @@ node --test tests/node/stream-tool-sieve.test.js - DSML `<|DSML|tool_calls>` wrapper 正常解析 - legacy canonical `` wrapper 正常解析 +- 别名变体（``、`<｜tool_calls>`、`<|tool_calls>`）正常解析 +- 混搭标签（DSML wrapper + canonical inner）归一化后正常解析 +- 波浪线围栏 `~~~` 内的示例不执行 +- 嵌套围栏（4 反引号嵌套 3 反引号）内的示例不执行 +- 文本 mention 标签名后紧跟真正工具调用的场景（含同一 wrapper 变体） - 非兼容内容按普通文本透传 - 代码块示例不执行 diff --git a/internal/js/helpers/stream-tool-sieve/parse.js b/internal/js/helpers/stream-tool-sieve/parse.js index fdae967..1d37b60 100644 --- a/internal/js/helpers/stream-tool-sieve/parse.js +++ b/internal/js/helpers/stream-tool-sieve/parse.js @@ -8,7 +8,7 @@ const { stripFencedCodeBlocks, } = require('./parse_payload'); -const TOOL_MARKUP_PREFIXES = [' 0) { + return out.slice(0, beforeFenceIdx).join(''); + } + return ''; + } + return out.join(''); +} + +function parseFenceOpenLine(trimmed) { + if (trimmed.length < 3) return null; + const ch = trimmed[0]; + if (ch !== '`' && ch !== '~') return null; + let count = 0; + while (count < trimmed.length && trimmed[count] === ch) count++; + if (count < 3) return null; + return { ch, count }; +} + +function isFenceCloseLine(trimmed, fenceChar, fenceLen) { + if (!fenceChar || !trimmed || trimmed[0] !== fenceChar) return false; + let count = 0; + while (count < trimmed.length && trimmed[count] === fenceChar) count++; + if (count < fenceLen) return false; + return trimmed.slice(count).trim() === ''; +} + +function cdataStartsBeforeFence(line) { + const cdataIdx = line.toLowerCase().indexOf('= 0 ? line.indexOf('```') : Infinity, + line.indexOf('~~~') >= 0 ? line.indexOf('~~~') : Infinity, + ); + return fenceIdx === Infinity || cdataIdx < fenceIdx; +} + +function updateCDATAStateLine(inCDATA, line) { + const lower = line.toLowerCase(); + let pos = 0; + let state = inCDATA; + while (pos < lower.length) { + if (state) { + const end = lower.indexOf(']]>', pos); + if (end < 0) return true; + pos = end + ']]>'.length; + state = false; + continue; + } + const start = lower.indexOf('). return { text: replaceDSMLToolMarkupOutsideIgnored(raw), ok: true, @@ -71,6 +166,24 @@ const DSML_TOOL_MARKUP_ALIASES = [ { from: '', to: '' }, { from: '<|dsml|parameter', to: '', to: '' }, + { from: '', to: '' }, + { from: '', to: '' }, + { from: '', to: '' }, + { from: '<|tool_calls', to: '', to: '' }, + { from: '<|invoke', to: '', to: '' }, + { from: '<|parameter', to: '', to: '' }, + { from: '<｜tool_calls', to: '', to: '' }, + { from: '<｜invoke', to: '', to: '' }, + { from: '<｜parameter', to: '', to: '' }, ]; const CANONICAL_TOOL_MARKUP_PREFIXES = [ @@ -190,7 +303,8 @@ function findXmlElementBlocks(text, tag) { } const end = findMatchingXmlEndTagOutsideCDATA(source, name, start.bodyStart); if (!end) { - break; + pos = start.bodyStart; + continue; } out.push({ attrs: start.attrs, diff --git a/internal/js/helpers/stream-tool-sieve/sieve-xml.js b/internal/js/helpers/stream-tool-sieve/sieve-xml.js index cff8fe7..300c2f0 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve-xml.js +++ b/internal/js/helpers/stream-tool-sieve/sieve-xml.js @@ -4,43 +4,77 @@ const { parseToolCalls } = require('./parse'); // XML wrapper tag pair used by the streaming sieve. const XML_TOOL_TAG_PAIRS = [ { open: '<|dsml|tool_calls', close: '' }, + { open: '' }, + { open: '<｜tool_calls', close: '' }, + { open: '<|tool_calls', close: '' }, { open: '' }, ]; -const XML_TOOL_OPENING_TAGS = XML_TOOL_TAG_PAIRS.map(p => p.open); +const XML_TOOL_OPENING_TAGS = [ + ...XML_TOOL_TAG_PAIRS.map(p => p.open), + '<|dsml|invoke', ' 0) { - const trimmedFence = trimWrappingJSONFence(prefixPart, suffixPart); - return { - ready: true, - prefix: trimmedFence.prefix, - calls: parsed, - suffix: trimmedFence.suffix, - }; + let searchFrom = 0; + while (searchFrom < lower.length) { + const openIdx = findXMLOpenOutsideCDATA(captured, pair.open, searchFrom); + if (openIdx < 0) { + break; + } + // Ignore closing tags that appear inside CDATA payloads, such as + // write-file content containing tool-call documentation examples. + const closeIdx = findMatchingXMLToolWrapperClose(captured, pair.open, pair.close, openIdx); + if (closeIdx < 0) { + anyOpenFound = true; + searchFrom = openIdx + pair.open.length; + continue; + } + const closeEnd = closeIdx + pair.close.length; + const xmlBlock = captured.slice(openIdx, closeEnd); + let prefixPart = captured.slice(0, openIdx); + let suffixPart = captured.slice(closeEnd); + const parsed = parseToolCalls(xmlBlock, toolNames); + if (Array.isArray(parsed) && parsed.length > 0) { + const trimmedFence = trimWrappingJSONFence(prefixPart, suffixPart); + if (!best || openIdx < best.start) { + best = { + start: openIdx, + prefix: trimmedFence.prefix, + calls: parsed, + suffix: trimmedFence.suffix, + }; + } + break; + } + if (!rejected || openIdx < rejected.start) { + rejected = { + start: openIdx, + prefix: prefixPart + xmlBlock, + suffix: suffixPart, + }; + } + searchFrom = openIdx + pair.open.length; } + } + if (best) { + return { ready: true, prefix: best.prefix, calls: best.calls, suffix: best.suffix }; + } + if (anyOpenFound) { + // At least one opening tag was found but none had a matching close tag. + return { ready: false, prefix: '', calls: [], suffix: '' }; + } + if (rejected) { // If this block failed to become a tool call, pass it through as text. - return { ready: true, prefix: prefixPart + xmlBlock, calls: [], suffix: suffixPart }; + return { ready: true, prefix: rejected.prefix, calls: [], suffix: rejected.suffix }; } if (!containsAnyToolCallWrapper(lower)) { const found = firstInvokeIndex(lower); @@ -70,6 +104,89 @@ function consumeXMLToolCapture(captured, toolNames, trimWrappingJSONFence) { return { ready: false, prefix: '', calls: [], suffix: '' }; } +function findMatchingXMLToolWrapperClose(s, openTag, closeTag, openIdx) { + const text = typeof s === 'string' ? s : ''; + const openTarget = String(openTag || '').toLowerCase(); + const closeTarget = String(closeTag || '').toLowerCase(); + if (!text || !openTarget || !closeTarget || openIdx < 0) { + return -1; + } + const lower = text.toLowerCase(); + let depth = 1; + for (let i = openIdx + openTarget.length; i < text.length;) { + if (lower.startsWith('', i + ''.length; + continue; + } + if (lower.startsWith('', i + ''.length; + continue; + } + if (lower.startsWith(closeTarget, i)) { + depth -= 1; + if (depth === 0) { + return i; + } + i += closeTarget.length; + continue; + } + if (lower.startsWith(openTarget, i) && hasXMLToolTagBoundary(text, i + openTarget.length)) { + depth += 1; + i += openTarget.length; + continue; + } + i += 1; + } + return -1; +} + +function findXMLOpenOutsideCDATA(s, openTag, start) { + const text = typeof s === 'string' ? s : ''; + const target = String(openTag || '').toLowerCase(); + if (!text || !target) { + return -1; + } + const lower = text.toLowerCase(); + for (let i = Math.max(0, start || 0); i < text.length;) { + if (lower.startsWith('', i + ''.length; + continue; + } + if (lower.startsWith('', i + ''.length; + continue; + } + if (lower.startsWith(target, i) && hasXMLToolTagBoundary(text, i + target.length)) { + return i; + } + i += 1; + } + return -1; +} + +function hasXMLToolTagBoundary(text, idx) { + if (idx >= text.length) { + return true; + } + return [' ', '\t', '\n', '\r', '>', '/'].includes(text[idx]); +} + function hasOpenXMLToolTag(captured) { const lower = captured.toLowerCase(); for (const pair of XML_TOOL_TAG_PAIRS) { @@ -84,12 +201,24 @@ function hasOpenXMLToolTag(captured) { } function containsAnyToolCallWrapper(lower) { - return lower.includes('= 0 && (dsmlIdx < 0 || idx < dsmlIdx)) { + dsmlIdx = idx; + } + } if (xmlIdx < 0) { return { index: dsmlIdx, dsml: dsmlIdx >= 0 }; } diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index 6ae85f7..8a31888 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -43,6 +43,10 @@ function processToolSieveChunk(state, chunk, toolNames) { resetIncrementalToolState(state); if (Array.isArray(consumed.calls) && consumed.calls.length > 0) { + if (consumed.prefix) { + noteText(state, consumed.prefix); + events.push({ type: 'text', text: consumed.prefix }); + } state.pendingToolRaw = captured; state.pendingToolCalls = consumed.calls; if (consumed.suffix) { diff --git a/internal/js/helpers/stream-tool-sieve/state.js b/internal/js/helpers/stream-tool-sieve/state.js index 447ecdf..f9fb2b5 100644 --- a/internal/js/helpers/stream-tool-sieve/state.js +++ b/internal/js/helpers/stream-tool-sieve/state.js @@ -7,6 +7,7 @@ function createToolSieveState() { capturing: false, codeFenceStack: [], codeFencePendingTicks: 0, + codeFencePendingTildes: 0, codeFenceLineStart: true, pendingToolRaw: '', pendingToolCalls: [], @@ -46,8 +47,7 @@ function insideCodeFence(text) { if (!t) { return false; } - const ticks = (t.match(/```/g) || []).length; - return ticks % 2 === 1; + return simulateCodeFenceState([], 0, 0, true, t).stack.length > 0; } function insideCodeFenceWithState(state, text) { @@ -57,6 +57,7 @@ function insideCodeFenceWithState(state, text) { const simulated = simulateCodeFenceState( Array.isArray(state.codeFenceStack) ? state.codeFenceStack : [], Number.isInteger(state.codeFencePendingTicks) ? state.codeFencePendingTicks : 0, + Number.isInteger(state.codeFencePendingTildes) ? state.codeFencePendingTildes : 0, state.codeFenceLineStart !== false, text, ); @@ -70,37 +71,57 @@ function updateCodeFenceState(state, text) { const next = simulateCodeFenceState( Array.isArray(state.codeFenceStack) ? state.codeFenceStack : [], Number.isInteger(state.codeFencePendingTicks) ? state.codeFencePendingTicks : 0, + Number.isInteger(state.codeFencePendingTildes) ? state.codeFencePendingTildes : 0, state.codeFenceLineStart !== false, text, ); state.codeFenceStack = next.stack; state.codeFencePendingTicks = next.pendingTicks; + state.codeFencePendingTildes = next.pendingTildes; state.codeFenceLineStart = next.lineStart; } -function simulateCodeFenceState(stack, pendingTicks, lineStart, text) { +function simulateCodeFenceState(stack, pendingTicks, pendingTildes, lineStart, text) { const chunk = typeof text === 'string' ? text : ''; const nextStack = Array.isArray(stack) ? [...stack] : []; let ticks = Number.isInteger(pendingTicks) ? pendingTicks : 0; + let tildes = Number.isInteger(pendingTildes) ? pendingTildes : 0; let atLineStart = lineStart !== false; - const flushTicks = () => { + const flushPending = () => { if (ticks > 0) { if (atLineStart && ticks >= 3) { - applyFenceMarker(nextStack, ticks); + applyFenceMarker(nextStack, ticks); // positive = backtick } atLineStart = false; ticks = 0; } + if (tildes > 0) { + if (atLineStart && tildes >= 3) { + applyFenceMarker(nextStack, -tildes); // negative = tilde + } + atLineStart = false; + tildes = 0; + } }; for (let i = 0; i < chunk.length; i += 1) { const ch = chunk[i]; if (ch === '`') { + if (tildes > 0) { + flushPending(); + } ticks += 1; continue; } - flushTicks(); + if (ch === '~') { + if (ticks > 0) { + flushPending(); + } + tildes += 1; + continue; + } + flushPending(); if (ch === '\n' || ch === '\r') { atLineStart = true; continue; @@ -110,29 +131,37 @@ function simulateCodeFenceState(stack, pendingTicks, lineStart, text) { } atLineStart = false; } - // keep ticks for cross-chunk continuation. return { stack: nextStack, pendingTicks: ticks, + pendingTildes: tildes, lineStart: atLineStart, }; } -function applyFenceMarker(stack, ticks) { +// Positive values = backtick fences, negative = tilde fences. +// Closing must match fence type. +function applyFenceMarker(stack, marker) { if (!Array.isArray(stack)) { return; } if (stack.length === 0) { - stack.push(ticks); + stack.push(marker); return; } const top = stack[stack.length - 1]; - if (ticks >= top) { + const sameType = (top > 0 && marker > 0) || (top < 0 && marker < 0); + if (!sameType) { + stack.push(marker); + return; + } + const absMarker = Math.abs(marker); + const absTop = Math.abs(top); + if (absMarker >= absTop) { stack.pop(); return; } - // nested/open inner fence using longer marker for robustness. - stack.push(ticks); + stack.push(marker); } function hasMeaningfulText(text) { diff --git a/internal/js/helpers/stream-tool-sieve/tool-keywords.js b/internal/js/helpers/stream-tool-sieve/tool-keywords.js index 5191c68..661abe4 100644 --- a/internal/js/helpers/stream-tool-sieve/tool-keywords.js +++ b/internal/js/helpers/stream-tool-sieve/tool-keywords.js @@ -2,16 +2,30 @@ const XML_TOOL_SEGMENT_TAGS = [ '<|dsml|tool_calls>', '<|dsml|tool_calls\n', '<|dsml|tool_calls ', + '<|dsml|invoke ', '<|dsml|invoke\n', '<|dsml|invoke\t', '<|dsml|invoke\r', + '', '', '<｜tool_calls\n', '<｜tool_calls ', + '<｜invoke ', '<｜invoke\n', '<｜invoke\t', '<｜invoke\r', + '<|tool_calls>', '<|tool_calls\n', '<|tool_calls ', + '<|invoke ', '<|invoke\n', '<|invoke\t', '<|invoke\r', '', '', + '', + '', + '', '', ]; diff --git a/internal/toolcall/fence_edge_test.go b/internal/toolcall/fence_edge_test.go new file mode 100644 index 0000000..5faff0b --- /dev/null +++ b/internal/toolcall/fence_edge_test.go @@ -0,0 +1,66 @@ +package toolcall + +import ( + "strings" + "testing" +) + +// 4 反引号嵌套 3 反引号 +func TestStripFencedCodeBlocks_NestedFourBackticks(t *testing.T) { + text := "Before\n\x60\x60\x60\x60markdown\nHere is \x60\x60\x60 nested \x60\x60\x60 example\n\x60\x60\x60\x60\nAfter" + got := stripFencedCodeBlocks(text) + if !strings.Contains(got, "Before") || !strings.Contains(got, "After") { + t.Fatalf("expected Before and After preserved, got %q", got) + } + if strings.Contains(got, "nested") { + t.Fatalf("expected nested content stripped, got %q", got) + } +} + +// 波浪线围栏 +func TestStripFencedCodeBlocks_TildeFence(t *testing.T) { + text := "Before\n~~~python\ncode here\n~~~\nAfter" + got := stripFencedCodeBlocks(text) + if !strings.Contains(got, "Before") || !strings.Contains(got, "After") { + t.Fatalf("expected Before/After, got %q", got) + } + if strings.Contains(got, "code here") { + t.Fatalf("expected code stripped, got %q", got) + } +} + +// 未闭合围栏 + 后面跟真正的工具调用：不应返回空字符串 +func TestStripFencedCodeBlocks_UnclosedFencePreservesToolCall(t *testing.T) { + text := "Example:\n\x60\x60\x60xml\nREADME.md\n\ngo" + got := stripFencedCodeBlocks(text) + if got == "" { + t.Fatalf("unclosed fence should not truncate everything — real tool call after the fence is lost") + } +} + +// CDATA 内的围栏不应被剥离 +func TestStripFencedCodeBlocks_FenceInsideCDATA(t *testing.T) { + text := "\n\n" + got := stripFencedCodeBlocks(text) + if !strings.Contains(got, "\x60\x60\x60python") { + t.Fatalf("fenced code inside CDATA should be preserved, got %q", got) + } +} + +// 连续多个围栏 +func TestStripFencedCodeBlocks_MultipleFences(t *testing.T) { + text := "Before\n\x60\x60\x60\nfence1\n\x60\x60\x60\nMiddle\n\x60\x60\x60\nfence2\n\x60\x60\x60\nAfter" + got := stripFencedCodeBlocks(text) + if !strings.Contains(got, "Before") || !strings.Contains(got, "Middle") || !strings.Contains(got, "After") { + t.Fatalf("expected non-fenced content preserved, got %q", got) + } +} + +// 围栏包含内嵌 ``` 行但没有独立成行 +func TestStripFencedCodeBlocks_InlineBackticksNotFence(t *testing.T) { + text := "Before\n\x60\x60\x60go\nfmt.Println(\x60\x60\x60hello\x60\x60\x60)\n\x60\x60\x60\nAfter" + got := stripFencedCodeBlocks(text) + if !strings.Contains(got, "Before") || !strings.Contains(got, "After") { + t.Fatalf("expected Before/After, got %q", got) + } +} diff --git a/internal/toolcall/toolcalls_dsml.go b/internal/toolcall/toolcalls_dsml.go index e694a00..df6cda2 100644 --- a/internal/toolcall/toolcalls_dsml.go +++ b/internal/toolcall/toolcalls_dsml.go @@ -6,13 +6,13 @@ func normalizeDSMLToolCallMarkup(text string) (string, bool) { if text == "" { return "", true } - hasDSML, hasCanonical := toolMarkupStylesOutsideIgnored(text) - if hasDSML && hasCanonical { - return text, false - } - if !hasDSML { + hasAliasLikeMarkup, _ := toolMarkupStylesOutsideIgnored(text) + if !hasAliasLikeMarkup { return text, true } + // Always normalize DSML aliases to canonical form, even when canonical + // tags coexist. Models frequently mix DSML wrapper tags with canonical + // inner tags (e.g., <｜tool_calls>). return replaceDSMLToolMarkupOutsideIgnored(text), true } @@ -26,6 +26,24 @@ var dsmlToolMarkupAliases = []struct { {"", ""}, {"<|dsml|parameter", "", ""}, + {"", ""}, + {"", ""}, + {"", ""}, + {"<|tool_calls", "", ""}, + {"<|invoke", "", ""}, + {"<|parameter", "", ""}, + {"<｜tool_calls", "", ""}, + {"<｜invoke", "", ""}, + {"<｜parameter", "", ""}, } var canonicalToolMarkupPrefixes = []string{ diff --git a/internal/toolcall/toolcalls_parse.go b/internal/toolcall/toolcalls_parse.go index 272127b..d9e99f9 100644 --- a/internal/toolcall/toolcalls_parse.go +++ b/internal/toolcall/toolcalls_parse.go @@ -93,7 +93,11 @@ func filterToolCallsDetailed(parsed []ParsedToolCall) ([]ParsedToolCall, []strin func looksLikeToolCallSyntax(text string) bool { lower := strings.ToLower(text) - return strings.Contains(lower, "<|dsml|tool_calls") || strings.Contains(lower, " 0 && beforeFenceLen <= len(result) { + return result[:beforeFenceLen] + } return "" } return b.String() diff --git a/internal/toolcall/toolcalls_parse_markup.go b/internal/toolcall/toolcalls_parse_markup.go index 9c4edd3..84ceeba 100644 --- a/internal/toolcall/toolcalls_parse_markup.go +++ b/internal/toolcall/toolcalls_parse_markup.go @@ -124,7 +124,8 @@ func findXMLElementBlocks(text, tag string) []xmlElementBlock { } closeStart, closeEnd, ok := findMatchingXMLEndTagOutsideCDATA(text, tag, bodyStart) if !ok { - break + pos = bodyStart + continue } out = append(out, xmlElementBlock{ Attrs: attrs, diff --git a/internal/toolcall/toolcalls_test.go b/internal/toolcall/toolcalls_test.go index ab6c8cd..187e79b 100644 --- a/internal/toolcall/toolcalls_test.go +++ b/internal/toolcall/toolcalls_test.go @@ -53,11 +53,16 @@ func TestParseToolCallsSupportsDSMLShellWithCanonicalExampleInCDATA(t *testing.T } } -func TestParseToolCallsRejectsMixedDSMLAndCanonicalToolTags(t *testing.T) { +func TestParseToolCallsNormalizesMixedDSMLAndCanonicalToolTags(t *testing.T) { + // Models commonly mix DSML wrapper tags with canonical inner tags. + // These should be normalized and parsed, not rejected. text := `<|DSML|tool_calls><|DSML|parameter name="command">pwd` calls := ParseToolCalls(text, []string{"Bash"}) - if len(calls) != 0 { - t.Fatalf("expected mixed DSML/XML tool tags to be rejected, got %#v", calls) + if len(calls) != 1 { + t.Fatalf("expected mixed DSML/XML tool tags to be normalized and parsed, got %#v", calls) + } + if calls[0].Name != "Bash" || calls[0].Input["command"] != "pwd" { + t.Fatalf("unexpected mixed DSML parse result: %#v", calls[0]) } } @@ -438,3 +443,25 @@ func TestParseToolCallsParsesAfterFourBacktickFence(t *testing.T) { t.Fatalf("expected non-fenced tool call to be parsed, got %#v", res.Calls[0]) } } + +func TestParseToolCallsSkipsProseMentionOfSameWrapperVariant(t *testing.T) { + text := strings.Join([]string{ + "Summary: support canonical and DSML <|DSML|tool_calls> wrappers.", + "", + "<|DSML|tool_calls>", + "<|DSML|invoke name=\"Bash\">", + "<|DSML|parameter name=\"command\">", + "", + "", + }, "\n") + res := ParseToolCallsDetailed(text, []string{"Bash"}) + if len(res.Calls) != 1 { + t.Fatalf("expected one parsed call after prose mention, got %#v", res.Calls) + } + if res.Calls[0].Name != "Bash" { + t.Fatalf("expected Bash call, got %#v", res.Calls[0]) + } + if got, _ := res.Calls[0].Input["command"].(string); got != "git status" { + t.Fatalf("expected command to parse, got %q", got) + } +} diff --git a/internal/toolstream/complex_edge_test.go b/internal/toolstream/complex_edge_test.go new file mode 100644 index 0000000..ec5664d --- /dev/null +++ b/internal/toolstream/complex_edge_test.go @@ -0,0 +1,556 @@ +package toolstream + +import ( + "strings" + "testing" +) + +// ---- 错位工具块 ---- + +// 只有 没有 +func TestSieve_MismatchedClose_OnlyClosingTag(t *testing.T) { + var state State + chunks := []string{ + "一些正文内容\n", + "\n", + "后续内容", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var text strings.Builder + tc := 0 + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 0 { + t.Fatalf("孤立闭合标签不应触发工具调用，got %d", tc) + } + if !strings.Contains(text.String(), "一些正文") || !strings.Contains(text.String(), "后续内容") { + t.Fatalf("应保留所有文本, got %q", text.String()) + } +} + +// 打开后跟的不是 而是普通文本 +func TestSieve_ToolCallsWrapperWithNoInvoke(t *testing.T) { + var state State + chunks := []string{ + "\n", + "这里没有 invoke 标签\n", + "\n", + "后续内容", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var text strings.Builder + tc := 0 + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 0 { + t.Fatalf("无 invoke 不应触发工具调用，got %d", tc) + } +} + +// 两个连续工具调用块 +func TestSieve_TwoConsecutiveToolCallBlocks(t *testing.T) { + var state State + chunks := []string{ + `a.txt`, + "\n", + `b.txt`, + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + tc := 0 + for _, e := range events { + tc += len(e.ToolCalls) + } + if tc != 2 { + t.Fatalf("应解析出两个工具调用，got %d, events=%#v", tc, events) + } +} + +// ---- 围栏内的工具调用不应触发 ---- + +// 反引号围栏内有完整工具调用 + 围栏外有真正的工具调用 +func TestSieve_FencedExampleThenRealToolCall(t *testing.T) { + var state State + chunks := []string{ + "示例：\n```xml\n", + `1`, + "\n```\n", + `real.txt`, + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file", "fake"})...) + } + events = append(events, Flush(&state, []string{"read_file", "fake"})...) + + var text strings.Builder + tc := 0 + var names []string + for _, e := range events { + text.WriteString(e.Content) + for _, call := range e.ToolCalls { + tc++ + names = append(names, call.Name) + } + } + if tc != 1 { + t.Fatalf("应只触发围栏外的工具调用，got %d, names=%v", tc, names) + } + if names[0] != "read_file" { + t.Fatalf("应触发 read_file，got %v", names) + } + if !strings.Contains(text.String(), "示例") { + t.Fatalf("围栏前文本应保留, got %q", text.String()) + } +} + +// 波浪线围栏包裹工具调用 +func TestSieve_TildeFencedToolCallIgnored(t *testing.T) { + var state State + chunks := []string{ + "~~~\n", + `x`, + "\n~~~\n", + "结束", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + tc := 0 + var text strings.Builder + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 0 { + t.Fatalf("波浪线围栏内工具调用不应触发，got %d", tc) + } + if !strings.Contains(text.String(), "结束") { + t.Fatalf("围栏后文本应保留, got %q", text.String()) + } +} + +// 4 反引号嵌套 3 反引号，内含工具标签 +func TestSieve_FourBacktickNestedThreeWithToolCall(t *testing.T) { + var state State + chunks := []string{ + "````markdown\n", + "```xml\n", + `x`, + "\n```\n", + "````\n", + "外部文本", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + tc := 0 + var text strings.Builder + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 0 { + t.Fatalf("4反引号嵌套内的工具调用不应触发，got %d", tc) + } + if !strings.Contains(text.String(), "外部文本") { + t.Fatalf("围栏外文本应保留, got %q", text.String()) + } +} + +// ---- DSML 变体在围栏内不触发 ---- + +func TestSieve_DSMLInsideFenceIgnored(t *testing.T) { + var state State + chunks := []string{ + "```\n", + "<|DSML|tool_calls>\n", + `<|DSML|invoke name="read_file">`, + `<|DSML|parameter name="path">x`, + "\n", + "\n", + "```\n", + "结束", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + tc := 0 + for _, e := range events { + tc += len(e.ToolCalls) + } + if tc != 0 { + t.Fatalf("围栏内的 DSML 工具调用不应触发，got %d", tc) + } +} + +// ---- 工具调用前后有丰富文本 ---- + +func TestSieve_RichTextAroundToolCall(t *testing.T) { + var state State + chunks := []string{ + "我来帮你查看文件内容。\n\n", + "首先读取 README：\n", + `README.md`, + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var text strings.Builder + tc := 0 + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 1 { + t.Fatalf("应有一个工具调用，got %d", tc) + } + if !strings.Contains(text.String(), "帮你查看") { + t.Fatalf("前置文本丢失, got %q", text.String()) + } + if strings.Contains(text.String(), "\n", + `` + "\n", + `test.md` + "\n", + `` + "\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"write_file"})...) + } + events = append(events, Flush(&state, []string{"write_file"})...) + + var text strings.Builder + tc := 0 + var gotContent any + for _, e := range events { + text.WriteString(e.Content) + if len(e.ToolCalls) > 0 { + tc += len(e.ToolCalls) + gotContent = e.ToolCalls[0].Input["content"] + } + } + if tc != 1 { + t.Fatalf("应有一个工具调用，got %d", tc) + } + content, _ := gotContent.(string) + if content != payload { + t.Fatalf("CDATA 内围栏内容应完整保留，got %q want %q", content, payload) + } + if text.Len() != 0 { + t.Fatalf("不应有文本泄漏, got %q", text.String()) + } +} + +// ---- 极端 token 拆分 ---- + +// 工具标签被拆成单字符流式到达 +func TestSieve_CharByCharToolCall(t *testing.T) { + var state State + full := `go.mod` + var events []Event + for _, ch := range full { + events = append(events, ProcessChunk(&state, string(ch), []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var text strings.Builder + tc := 0 + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 1 { + t.Fatalf("单字符流式应解析出工具调用，got %d", tc) + } + if strings.Contains(text.String(), "invoke") { + t.Fatalf("标签泄漏, got %q", text.String()) + } +} + +// ---- 混合格式变体 ---- + +// 全宽竖线 wrapper + DSML invoke +func TestSieve_FullwidthPipeWrapperDSMLInvoke(t *testing.T) { + var state State + chunks := []string{ + "<｜tool_calls>\n", + "<|DSML|invoke name=\"read_file\">\n", + "<|DSML|parameter name=\"path\">README.md\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var text strings.Builder + tc := 0 + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + if tc != 1 { + t.Fatalf("全宽+DSML混合应解析成功，got %d", tc) + } + if strings.Contains(strings.ToLower(text.String()), "dsml") { + t.Fatalf("DSML 标签泄漏, got %q", text.String()) + } +} + +// ---- 未闭合工具块应回退为文本 ---- + +func TestSieve_UnclosedToolCallBlockFallsBack(t *testing.T) { + var state State + chunks := []string{ + "\n", + `` + "\n", + `README.md` + "\n", + // 缺少 和 + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var text strings.Builder + tc := 0 + for _, e := range events { + text.WriteString(e.Content) + tc += len(e.ToolCalls) + } + // 未闭合的应回退为文本，不应丢失 + if text.String() == "" { + t.Fatalf("未闭合工具块不应丢失所有内容") + } + if tc != 0 { + t.Fatalf("未闭合工具块不应解析出工具调用，got %d", tc) + } +} + +// ---- 文本中 mention 标签变体名 + 真正的工具调用 ---- + +// 模型输出 commit message 文本中包含 等 mention， +// 紧随其后是真正的 DSML 工具调用。mention 的变体和实际工具调用变体不同。 +func TestSieve_TagMentionInTextThenRealToolCall(t *testing.T) { + var state State + chunks := []string{ + "建议的 commit message：\n\nfeat: expand DSML alias support\n\n", + "Add support for , ", + "<｜tool_calls> (fullwidth pipe),\n", + "and <|tool_calls> wrapper variants.\n\n", + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"Bash\">\n", + "<|DSML|parameter name=\"command\">\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"Bash"})...) + } + events = append(events, Flush(&state, []string{"Bash"})...) + + var text strings.Builder + tc := 0 + var names []string + for _, e := range events { + text.WriteString(e.Content) + for _, call := range e.ToolCalls { + tc++ + names = append(names, call.Name) + } + } + + if tc != 1 { + t.Fatalf("应解析出 1 个工具调用，got %d, text=%q", tc, text.String()) + } + if names[0] != "Bash" { + t.Fatalf("应解析出 Bash，got %v", names) + } + if !strings.Contains(text.String(), "commit message") { + t.Fatalf("前置文本应保留, got %q", text.String()) + } +} + +func TestSieve_SameVariantTagMentionInTextThenRealToolCall(t *testing.T) { + var state State + chunks := []string{ + "Summary: support canonical and DSML <|DSML|tool_calls> wrappers.\n\n", + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"Bash\">\n", + "<|DSML|parameter name=\"command\">\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"Bash"})...) + } + events = append(events, Flush(&state, []string{"Bash"})...) + + var text strings.Builder + var callName string + var command string + callCount := 0 + for _, e := range events { + text.WriteString(e.Content) + for _, call := range e.ToolCalls { + callCount++ + callName = call.Name + command, _ = call.Input["command"].(string) + } + } + + if callCount != 1 { + t.Fatalf("应解析出 1 个工具调用，got %d, text=%q", callCount, text.String()) + } + if callName != "Bash" { + t.Fatalf("应解析出 Bash，got %q", callName) + } + if command != "git status" { + t.Fatalf("应解析出 command，got %q", command) + } + if !strings.Contains(text.String(), "Summary:") { + t.Fatalf("前置文本应保留, got %q", text.String()) + } +} + +func TestSieve_ReviewSampleWithAliasMentionsPreservesBodyAndToolCalls(t *testing.T) { + var state State + chunks := []string{ + "Done reviewing the diff. Here's my analysis before we commit:\n\n", + "Summary of Changes\n", + "DSML wrapper variant support — recognize aliases (, <|tool_calls>, <｜tool_calls>) alongside canonical and <|DSML|tool_calls> wrappers.\n\n", + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"Bash\">\n", + "<|DSML|parameter name=\"command\">\n", + "<|DSML|parameter name=\"description\">\n", + "\n", + "<|DSML|invoke name=\"Bash\">\n", + "<|DSML|parameter name=\"command\">, <|tool_calls>, <｜tool_calls> alongside existing canonical wrappers.\nEOF\n)\"]]>\n", + "<|DSML|parameter name=\"description\">\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"Bash"})...) + } + events = append(events, Flush(&state, []string{"Bash"})...) + + var text strings.Builder + var commands []string + for _, e := range events { + text.WriteString(e.Content) + for _, call := range e.ToolCalls { + if call.Name == "Bash" { + cmd, _ := call.Input["command"].(string) + commands = append(commands, cmd) + } + } + } + + if len(commands) != 2 { + t.Fatalf("应解析出 2 个 Bash 工具调用，got %d, text=%q", len(commands), text.String()) + } + if !strings.Contains(text.String(), "<|DSML|tool_calls> wrappers") { + t.Fatalf("正文中的 DSML mention 应保留, got %q", text.String()) + } + if !strings.Contains(text.String(), "Summary of Changes") { + t.Fatalf("前置正文应完整保留, got %q", text.String()) + } + if strings.Contains(text.String(), "git add docs/toolcall-semantics.md") { + t.Fatalf("真实工具参数不应泄漏到正文, got %q", text.String()) + } + if !strings.Contains(commands[0], "git add") || !strings.Contains(commands[1], "git commit") { + t.Fatalf("工具参数解析不符合预期, got %#v", commands) + } +} + +func TestSieve_ChineseReviewSamplePreservesInlineDSMLMention(t *testing.T) { + var state State + chunks := []string{ + "# Context from my IDE setup:\n\n## My request for Codex:\n", + "基于我的审查，这是工作区更改的总结和提交。\n\n## 审查报告\n\n### 文档\n\nAPI.md 中的工具调用部分缺少针对新 DSML 别名的更新——它只提到了 `", + "<|DSML|tool_calls>` 和 canonical ``。由于这涉及 API 兼容性和文档准确性，需要在下游进行记录。\n\n", + "### 代码\n\n所有更改现在一致地处理四个 DSML wrapper 变体。\n\n现在提交已暂存的更改。\n\n", + "<|DSML|tool_calls>\n", + " <|DSML|invoke name=\"Bash\">\n", + " <|DSML|parameter name=\"command\">\n", + " <|DSML|parameter name=\"description\">\n", + " \n", + "\n\n补充", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"Bash"})...) + } + events = append(events, Flush(&state, []string{"Bash"})...) + + var text strings.Builder + callCount := 0 + for _, e := range events { + text.WriteString(e.Content) + callCount += len(e.ToolCalls) + } + + if callCount != 1 { + t.Fatalf("应解析出 1 个工具调用，got %d, text=%q", callCount, text.String()) + } + want := "它只提到了 `<|DSML|tool_calls>` 和 canonical ``。由于这涉及 API 兼容性" + if !strings.Contains(text.String(), want) { + t.Fatalf("正文不应在 inline DSML mention 处截断, want contains %q, got %q", want, text.String()) + } + if !strings.Contains(text.String(), "补充") { + t.Fatalf("工具块后的正文应保留, got %q", text.String()) + } + if strings.Contains(text.String(), "<|DSML|invoke") { + t.Fatalf("真实工具块不应泄漏到正文, got %q", text.String()) + } +} diff --git a/internal/toolstream/fence_edge_sieve_test.go b/internal/toolstream/fence_edge_sieve_test.go new file mode 100644 index 0000000..d56335f --- /dev/null +++ b/internal/toolstream/fence_edge_sieve_test.go @@ -0,0 +1,59 @@ +package toolstream + +import ( + "strings" + "testing" +) + +// 波浪线围栏内的工具调用标签不应触发工具调用 +func TestProcessToolSieveTildeFenceDoesNotTriggerToolCall(t *testing.T) { + var state State + chunks := []string{ + "示例：\n~~~xml\n", + "README.md\n", + "~~~\n", + "完毕。", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var textContent strings.Builder + toolCalls := 0 + for _, evt := range events { + textContent.WriteString(evt.Content) + toolCalls += len(evt.ToolCalls) + } + + if toolCalls != 0 { + t.Fatalf("expected tilde-fenced tool example to stay text, got %d tool calls", toolCalls) + } + if !strings.Contains(textContent.String(), "示例") || !strings.Contains(textContent.String(), "完毕") { + t.Fatalf("expected surrounding text preserved, got %q", textContent.String()) + } +} + +// 4 反引号嵌套 3 反引号（内含工具标签）不应触发 +func TestProcessToolSieveNestedFourBacktickFenceDoesNotTrigger(t *testing.T) { + var state State + input := "说明：\n````xml\n```\nx\n```\n````\n结束。" + chunks := strings.SplitAfter(input, "\n") + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"read_file"})...) + } + events = append(events, Flush(&state, []string{"read_file"})...) + + var textContent strings.Builder + toolCalls := 0 + for _, evt := range events { + textContent.WriteString(evt.Content) + toolCalls += len(evt.ToolCalls) + } + + if toolCalls != 0 { + t.Fatalf("expected 4-backtick fenced example to stay text, got %d tool calls", toolCalls) + } +} diff --git a/internal/toolstream/tool_sieve_state.go b/internal/toolstream/tool_sieve_state.go index 1d709bd..cdc70d5 100644 --- a/internal/toolstream/tool_sieve_state.go +++ b/internal/toolstream/tool_sieve_state.go @@ -6,21 +6,22 @@ import ( ) type State struct { - pending strings.Builder - capture strings.Builder - capturing bool - codeFenceStack []int - codeFencePendingTicks int - codeFenceLineStart bool - pendingToolRaw string - pendingToolCalls []toolcall.ParsedToolCall - disableDeltas bool - toolNameSent bool - toolName string - toolArgsStart int - toolArgsSent int - toolArgsString bool - toolArgsDone bool + pending strings.Builder + capture strings.Builder + capturing bool + codeFenceStack []int + codeFencePendingTicks int + codeFencePendingTildes int + codeFenceNotLineStart bool // inverted: zero-value false means "at line start" + pendingToolRaw string + pendingToolCalls []toolcall.ParsedToolCall + disableDeltas bool + toolNameSent bool + toolName string + toolArgsStart int + toolArgsSent int + toolArgsString bool + toolArgsDone bool } type Event struct { @@ -63,7 +64,8 @@ func insideCodeFenceWithState(state *State, text string) bool { simulated := simulateCodeFenceState( state.codeFenceStack, state.codeFencePendingTicks, - state.codeFenceLineStart, + state.codeFencePendingTildes, + !state.codeFenceNotLineStart, text, ) return len(simulated.stack) > 0 @@ -73,7 +75,7 @@ func insideCodeFence(text string) bool { if text == "" { return false } - return len(simulateCodeFenceState(nil, 0, true, text).stack) > 0 + return len(simulateCodeFenceState(nil, 0, 0, true, text).stack) > 0 } func updateCodeFenceState(state *State, text string) { @@ -83,43 +85,65 @@ func updateCodeFenceState(state *State, text string) { next := simulateCodeFenceState( state.codeFenceStack, state.codeFencePendingTicks, - state.codeFenceLineStart, + state.codeFencePendingTildes, + !state.codeFenceNotLineStart, text, ) state.codeFenceStack = next.stack state.codeFencePendingTicks = next.pendingTicks - state.codeFenceLineStart = next.lineStart + state.codeFencePendingTildes = next.pendingTildes + state.codeFenceNotLineStart = !next.lineStart } type codeFenceSimulation struct { - stack []int - pendingTicks int - lineStart bool + stack []int + pendingTicks int + pendingTildes int + lineStart bool } -func simulateCodeFenceState(stack []int, pendingTicks int, lineStart bool, text string) codeFenceSimulation { +func simulateCodeFenceState(stack []int, pendingTicks, pendingTildes int, lineStart bool, text string) codeFenceSimulation { chunk := text nextStack := append([]int(nil), stack...) ticks := pendingTicks + tildes := pendingTildes atLineStart := lineStart - flushTicks := func() { + flushPending := func() { if ticks > 0 { if atLineStart && ticks >= 3 { - applyFenceMarker(&nextStack, ticks) + applyFenceMarker(&nextStack, ticks) // positive = backtick } atLineStart = false ticks = 0 } + if tildes > 0 { + if atLineStart && tildes >= 3 { + applyFenceMarker(&nextStack, -tildes) // negative = tilde + } + atLineStart = false + tildes = 0 + } } for i := 0; i < len(chunk); i++ { ch := chunk[i] if ch == '`' { + if tildes > 0 { + // Mixed chars — flush tildes first. + flushPending() + } ticks++ continue } - flushTicks() + if ch == '~' { + if ticks > 0 { + flushPending() + } + tildes++ + continue + } + flushPending() switch ch { case '\n', '\r': atLineStart = true @@ -134,24 +158,43 @@ func simulateCodeFenceState(stack []int, pendingTicks int, lineStart bool, text } return codeFenceSimulation{ - stack: nextStack, - pendingTicks: ticks, - lineStart: atLineStart, + stack: nextStack, + pendingTicks: ticks, + pendingTildes: tildes, + lineStart: atLineStart, } } -func applyFenceMarker(stack *[]int, ticks int) { - if stack == nil || ticks <= 0 { +// applyFenceMarker pushes or pops a fence marker on the stack. +// Positive values represent backtick fences, negative represent tilde fences. +// A closing marker must match the sign (type) of the opening marker. +func applyFenceMarker(stack *[]int, marker int) { + if stack == nil || marker == 0 { return } if len(*stack) == 0 { - *stack = append(*stack, ticks) + *stack = append(*stack, marker) return } top := (*stack)[len(*stack)-1] - if ticks >= top { + // Signs must match: backtick closes backtick, tilde closes tilde. + sameType := (top > 0 && marker > 0) || (top < 0 && marker < 0) + if !sameType { + // Different fence type — treat as nested. + *stack = append(*stack, marker) + return + } + absMarker := marker + absTop := top + if absMarker < 0 { + absMarker = -absMarker + } + if absTop < 0 { + absTop = -absTop + } + if absMarker >= absTop { *stack = (*stack)[:len(*stack)-1] return } - *stack = append(*stack, ticks) + *stack = append(*stack, marker) } diff --git a/internal/toolstream/tool_sieve_xml.go b/internal/toolstream/tool_sieve_xml.go index ae6de19..67d6555 100644 --- a/internal/toolstream/tool_sieve_xml.go +++ b/internal/toolstream/tool_sieve_xml.go @@ -9,13 +9,22 @@ import ( // --- XML tool call support for the streaming sieve --- //nolint:unused // kept as explicit tag inventory for future XML sieve refinements. -var xmlToolCallClosingTags = []string{"", ""} -var xmlToolCallOpeningTags = []string{"", "", "", "", ""} +var xmlToolCallOpeningTags = []string{ + ""}, + {""}, + {"<｜tool_calls", ""}, + {"<|tool_calls", ""}, {""}, } @@ -28,38 +37,80 @@ var xmlToolCallBlockPattern = regexp.MustCompile(`(?is)((?:", "<|dsml|tool_calls\n", "<|dsml|tool_calls ", "<|dsml|invoke ", "<|dsml|invoke\n", "<|dsml|invoke\t", "<|dsml|invoke\r", + "", "", "<｜tool_calls\n", "<｜tool_calls ", + "<｜invoke ", "<｜invoke\n", "<｜invoke\t", "<｜invoke\r", + "<|tool_calls>", "<|tool_calls\n", "<|tool_calls ", + "<|invoke ", "<|invoke\n", "<|invoke\t", "<|invoke\r", "", ". - closeIdx := findXMLCloseOutsideCDATA(captured, pair.close, openIdx+len(pair.open)) - if closeIdx < 0 { - // Opening tag is present but its specific closing tag hasn't arrived. - // Return not-ready so we keep buffering until the canonical wrapper closes. - return "", nil, "", false - } - closeEnd := closeIdx + len(pair.close) + anyOpenFound := false + type candidate struct { + start int + prefix string + calls []toolcall.ParsedToolCall + suffix string + } + type rejectedBlock struct { + start int + prefix string + suffix string + } + var best *candidate + var rejected *rejectedBlock - xmlBlock := captured[openIdx:closeEnd] - prefixPart := captured[:openIdx] - suffixPart := captured[closeEnd:] - parsed := toolcall.ParseToolCalls(xmlBlock, toolNames) - if len(parsed) > 0 { - prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart) - return prefixPart, parsed, suffixPart, true + // Scan every wrapper occurrence. Prose can mention a wrapper tag before the + // actual tool block, including the same variant as the real block. + for _, pair := range xmlToolCallTagPairs { + searchFrom := 0 + for searchFrom < len(lower) { + openIdx := findXMLOpenOutsideCDATA(captured, pair.open, searchFrom) + if openIdx < 0 { + break + } + // Find the matching closing tag outside CDATA. Long write-file tool + // calls often contain XML examples in CDATA, including . + closeIdx := findMatchingXMLToolWrapperClose(captured, pair.open, pair.close, openIdx) + if closeIdx < 0 { + anyOpenFound = true + searchFrom = openIdx + len(pair.open) + continue + } + closeEnd := closeIdx + len(pair.close) + + xmlBlock := captured[openIdx:closeEnd] + prefixPart := captured[:openIdx] + suffixPart := captured[closeEnd:] + parsed := toolcall.ParseToolCalls(xmlBlock, toolNames) + if len(parsed) > 0 { + prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart) + if best == nil || openIdx < best.start { + best = &candidate{start: openIdx, prefix: prefixPart, calls: parsed, suffix: suffixPart} + } + break + } + if rejected == nil || openIdx < rejected.start { + rejected = &rejectedBlock{start: openIdx, prefix: prefixPart + xmlBlock, suffix: suffixPart} + } + searchFrom = openIdx + len(pair.open) } + } + if best != nil { + return best.prefix, best.calls, best.suffix, true + } + if anyOpenFound { + // At least one opening tag was found but none had a matching close tag. + // Keep buffering until a closing tag arrives. + return "", nil, "", false + } + if rejected != nil { // If this block failed to become a tool call, pass it through as text. - return prefixPart + xmlBlock, nil, suffixPart, true + return rejected.prefix, nil, rejected.suffix, true } if !containsAnyToolCallWrapper(lower) { invokeIdx, dsml := firstInvokeIndex(lower) @@ -86,6 +137,88 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string, return "", nil, "", false } +func findMatchingXMLToolWrapperClose(s, openTag, closeTag string, openIdx int) int { + if s == "" || openTag == "" || closeTag == "" || openIdx < 0 { + return -1 + } + lower := strings.ToLower(s) + openTarget := strings.ToLower(openTag) + closeTarget := strings.ToLower(closeTag) + depth := 1 + for i := openIdx + len(openTarget); i < len(s); { + switch { + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], closeTarget): + depth-- + if depth == 0 { + return i + } + i += len(closeTarget) + case strings.HasPrefix(lower[i:], openTarget) && hasXMLToolTagBoundary(s, i+len(openTarget)): + depth++ + i += len(openTarget) + default: + i++ + } + } + return -1 +} + +func findXMLOpenOutsideCDATA(s, openTag string, start int) int { + if s == "" || openTag == "" { + return -1 + } + if start < 0 { + start = 0 + } + lower := strings.ToLower(s) + target := strings.ToLower(openTag) + for i := start; i < len(s); { + switch { + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], target) && hasXMLToolTagBoundary(s, i+len(target)): + return i + default: + i++ + } + } + return -1 +} + +func hasXMLToolTagBoundary(text string, idx int) bool { + if idx >= len(text) { + return true + } + switch text[idx] { + case ' ', '\t', '\n', '\r', '>', '/': + return true + default: + return false + } +} + // hasOpenXMLToolTag returns true if captured text contains an XML tool opening tag // whose SPECIFIC closing tag has not appeared yet. func hasOpenXMLToolTag(captured string) bool { @@ -144,12 +277,24 @@ func shouldKeepBareInvokeCapture(captured string) bool { } func containsAnyToolCallWrapper(lower string) bool { - return strings.Contains(lower, "= 0 && (dsmlIdx < 0 || idx < dsmlIdx) { + dsmlIdx = idx + } + } switch { case xmlIdx < 0: return dsmlIdx, dsmlIdx >= 0 diff --git a/internal/toolstream/tool_sieve_xml_test.go b/internal/toolstream/tool_sieve_xml_test.go index e35035f..0b9a7bb 100644 --- a/internal/toolstream/tool_sieve_xml_test.go +++ b/internal/toolstream/tool_sieve_xml_test.go @@ -678,3 +678,96 @@ func TestProcessToolSieveRepairsMissingOpeningWrapperWithoutLeakingInvokeText(t t.Fatalf("expected repaired missing-wrapper stream not to leak xml text, got %q", textContent.String()) } } + +// Test fullwidth pipe variant: <｜tool_calls> (U+FF5C) should be buffered and parsed. +func TestProcessToolSieveFullwidthPipeVariantDoesNotLeak(t *testing.T) { + var state State + chunks := []string{ + "<\uff5ctool_calls>\n", + "\n", + "git status\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"execute_command"})...) + } + events = append(events, Flush(&state, []string{"execute_command"})...) + + var textContent string + var toolCalls int + for _, evt := range events { + textContent += evt.Content + toolCalls += len(evt.ToolCalls) + } + + if strings.Contains(textContent, "invoke") || strings.Contains(textContent, "execute_command") { + t.Fatalf("fullwidth pipe variant leaked to text: %q", textContent) + } + if toolCalls != 1 { + t.Fatalf("expected one tool call from fullwidth pipe variant, got %d events=%#v", toolCalls, events) + } +} + +// Test with <|DSML|invoke> (DSML prefix without leading pipe on wrapper). +func TestProcessToolSieveDSMLPrefixVariantDoesNotLeak(t *testing.T) { + var state State + chunks := []string{ + "\n", + " <|DSML|invoke name=\"execute_command\">\n", + " <|DSML|parameter name=\"command\">\n", + " \n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"execute_command"})...) + } + events = append(events, Flush(&state, []string{"execute_command"})...) + + var textContent string + var toolCalls int + for _, evt := range events { + textContent += evt.Content + toolCalls += len(evt.ToolCalls) + } + + if strings.Contains(strings.ToLower(textContent), "dsml") || strings.Contains(textContent, "execute_command") { + t.Fatalf("DSML prefix variant leaked to text: %q", textContent) + } + if toolCalls != 1 { + t.Fatalf("expected one tool call from DSML prefix variant, got %d events=%#v", toolCalls, events) + } +} + +// Test with (no pipe anywhere) should be buffered and parsed. +func TestProcessToolSieveDSMLBarePrefixVariantDoesNotLeak(t *testing.T) { + var state State + chunks := []string{ + "\n", + "\n", + "\n", + "\n", + "", + } + var events []Event + for _, c := range chunks { + events = append(events, ProcessChunk(&state, c, []string{"execute_command"})...) + } + events = append(events, Flush(&state, []string{"execute_command"})...) + + var textContent string + var toolCalls int + for _, evt := range events { + textContent += evt.Content + toolCalls += len(evt.ToolCalls) + } + + if strings.Contains(strings.ToLower(textContent), "dsml") || strings.Contains(textContent, "execute_command") { + t.Fatalf("DSML bare prefix variant leaked to text: %q", textContent) + } + if toolCalls != 1 { + t.Fatalf("expected one tool call from DSML bare prefix variant, got %d events=%#v", toolCalls, events) + } +} diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index 40a6e42..d870d73 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -66,10 +66,90 @@ test('parseToolCalls keeps canonical XML examples inside DSML CDATA', () => { assert.deepEqual(calls[0].input, { path: 'notes.md', content }); }); -test('parseToolCalls rejects mixed DSML and XML tool tags', () => { +test('parseToolCalls normalizes mixed DSML and XML tool tags', () => { + // Models commonly mix DSML wrapper tags with canonical inner tags. const payload = '<|DSML|tool_calls><|DSML|parameter name="path">README.MD'; const calls = parseToolCalls(payload, ['read_file']); - assert.equal(calls.length, 0); + assert.equal(calls.length, 1); + assert.equal(calls[0].name, 'read_file'); + assert.deepEqual(calls[0].input, { path: 'README.MD' }); +}); + +test('parseToolCalls skips prose mention of same wrapper variant', () => { + const payload = [ + 'Summary: support canonical and DSML <|DSML|tool_calls> wrappers.', + '', + '<|DSML|tool_calls>', + '<|DSML|invoke name="Bash">', + '<|DSML|parameter name="command">', + '', + '', + ].join('\n'); + const calls = parseToolCalls(payload, ['Bash']); + assert.equal(calls.length, 1); + assert.equal(calls[0].name, 'Bash'); + assert.equal(calls[0].input.command, 'git status'); +}); + +test('sieve emits tool_calls after prose mentions same wrapper variant', () => { + const events = runSieve([ + 'Summary: support canonical and DSML <|DSML|tool_calls> wrappers.\n\n', + '<|DSML|tool_calls>\n', + '<|DSML|invoke name="Bash">\n', + '<|DSML|parameter name="command">\n', + '\n', + '', + ], ['Bash']); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 1); + assert.equal(finalCalls[0].name, 'Bash'); + assert.equal(finalCalls[0].input.command, 'git status'); + assert.equal(collectText(events).includes('Summary:'), true); +}); + +test('sieve preserves review body with alias mentions before real DSML tool calls', () => { + const events = runSieve([ + "Done reviewing the diff. Here's my analysis before we commit:\n\n", + 'Summary of Changes\n', + 'DSML wrapper variant support — recognize aliases (, <|tool_calls>, <｜tool_calls>) alongside canonical and <|DSML|tool_calls> wrappers.\n\n', + '<|DSML|tool_calls>\n', + '<|DSML|invoke name="Bash">\n', + '<|DSML|parameter name="command">\n', + '<|DSML|parameter name="description">\n', + '\n', + '<|DSML|invoke name="Bash">\n', + '<|DSML|parameter name="command">, <|tool_calls>, <｜tool_calls> alongside existing canonical wrappers.\nEOF\n)"]]>\n', + '<|DSML|parameter name="description">\n', + '\n', + '', + ], ['Bash']); + const text = collectText(events); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 2); + assert.equal(text.includes('<|DSML|tool_calls> wrappers'), true); + assert.equal(text.includes('Summary of Changes'), true); + assert.equal(text.includes('git add docs/toolcall-semantics.md'), false); +}); + +test('sieve preserves Chinese review body with inline DSML mention before real tool call', () => { + const events = runSieve([ + '# Context from my IDE setup:\n\n## My request for Codex:\n', + '基于我的审查，这是工作区更改的总结和提交。\n\n## 审查报告\n\n### 文档\n\nAPI.md 中的工具调用部分缺少针对新 DSML 别名的更新——它只提到了 `', + '<|DSML|tool_calls>` 和 canonical ``。由于这涉及 API 兼容性和文档准确性，需要在下游进行记录。\n\n', + '### 代码\n\n所有更改现在一致地处理四个 DSML wrapper 变体。\n\n现在提交已暂存的更改。\n\n', + '<|DSML|tool_calls>\n', + ' <|DSML|invoke name="Bash">\n', + ' <|DSML|parameter name="command">\n', + ' <|DSML|parameter name="description">\n', + ' \n', + '\n\n补充', + ], ['Bash']); + const text = collectText(events); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 1); + assert.equal(text.includes('它只提到了 `<|DSML|tool_calls>` 和 canonical ``。由于这涉及 API 兼容性'), true); + assert.equal(text.includes('补充'), true); + assert.equal(text.includes('<|DSML|invoke'), false); }); test('parseToolCalls ignores JSON tool_calls payload (XML-only)', () => {