mirror of
https://github.com/CJackHwang/ds2api.git
synced 2026-05-13 04:38:00 +08:00
feat: expand DSML tool-call alias and fence handling
Add support for DSML wrapper aliases (<dsml|tool_calls>, <|tool_calls>, <｜tool_calls>) alongside canonical XML. Normalize mixed DSML/canonical tags instead of rejecting them. Add tilde fence (~~~) support, fix nested fence and unclosed fence handling, support CDATA-protected fence content, and skip prose mentions when scanning for real tool blocks. Mirror all changes between Go and Node.js runtimes. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
CJACK
556
internal/toolstream/complex_edge_test.go
Normal file
556
internal/toolstream/complex_edge_test.go
Normal file
@@ -0,0 +1,556 @@
|
||||
package toolstream
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ---- 错位工具块 ----
|
||||
|
||||
// 只有 </tool_calls> 没有 <tool_calls>
|
||||
func TestSieve_MismatchedClose_OnlyClosingTag(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"一些正文内容\n",
|
||||
"</tool_calls>\n",
|
||||
"后续内容",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 0 {
|
||||
t.Fatalf("孤立闭合标签不应触发工具调用，got %d", tc)
|
||||
}
|
||||
if !strings.Contains(text.String(), "一些正文") || !strings.Contains(text.String(), "后续内容") {
|
||||
t.Fatalf("应保留所有文本, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// <tool_calls> 打开后跟的不是 <invoke> 而是普通文本
|
||||
func TestSieve_ToolCallsWrapperWithNoInvoke(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"<tool_calls>\n",
|
||||
"这里没有 invoke 标签\n",
|
||||
"</tool_calls>\n",
|
||||
"后续内容",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 0 {
|
||||
t.Fatalf("无 invoke 不应触发工具调用，got %d", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// 两个连续工具调用块
|
||||
func TestSieve_TwoConsecutiveToolCallBlocks(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
`<tool_calls><invoke name="read_file"><parameter name="path">a.txt</parameter></invoke></tool_calls>`,
|
||||
"\n",
|
||||
`<tool_calls><invoke name="read_file"><parameter name="path">b.txt</parameter></invoke></tool_calls>`,
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 2 {
|
||||
t.Fatalf("应解析出两个工具调用，got %d, events=%#v", tc, events)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 围栏内的工具调用不应触发 ----
|
||||
|
||||
// 反引号围栏内有完整工具调用 + 围栏外有真正的工具调用
|
||||
func TestSieve_FencedExampleThenRealToolCall(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"示例：\n```xml\n",
|
||||
`<tool_calls><invoke name="fake"><parameter name="x">1</parameter></invoke></tool_calls>`,
|
||||
"\n```\n",
|
||||
`<tool_calls><invoke name="read_file"><parameter name="path">real.txt</parameter></invoke></tool_calls>`,
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file", "fake"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file", "fake"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
var names []string
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
for _, call := range e.ToolCalls {
|
||||
tc++
|
||||
names = append(names, call.Name)
|
||||
}
|
||||
}
|
||||
if tc != 1 {
|
||||
t.Fatalf("应只触发围栏外的工具调用，got %d, names=%v", tc, names)
|
||||
}
|
||||
if names[0] != "read_file" {
|
||||
t.Fatalf("应触发 read_file，got %v", names)
|
||||
}
|
||||
if !strings.Contains(text.String(), "示例") {
|
||||
t.Fatalf("围栏前文本应保留, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 波浪线围栏包裹工具调用
|
||||
func TestSieve_TildeFencedToolCallIgnored(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"~~~\n",
|
||||
`<tool_calls><invoke name="read_file"><parameter name="path">x</parameter></invoke></tool_calls>`,
|
||||
"\n~~~\n",
|
||||
"结束",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
tc := 0
|
||||
var text strings.Builder
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 0 {
|
||||
t.Fatalf("波浪线围栏内工具调用不应触发，got %d", tc)
|
||||
}
|
||||
if !strings.Contains(text.String(), "结束") {
|
||||
t.Fatalf("围栏后文本应保留, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 4 反引号嵌套 3 反引号，内含工具标签
|
||||
func TestSieve_FourBacktickNestedThreeWithToolCall(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"````markdown\n",
|
||||
"```xml\n",
|
||||
`<tool_calls><invoke name="read_file"><parameter name="path">x</parameter></invoke></tool_calls>`,
|
||||
"\n```\n",
|
||||
"````\n",
|
||||
"外部文本",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
tc := 0
|
||||
var text strings.Builder
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 0 {
|
||||
t.Fatalf("4反引号嵌套内的工具调用不应触发，got %d", tc)
|
||||
}
|
||||
if !strings.Contains(text.String(), "外部文本") {
|
||||
t.Fatalf("围栏外文本应保留, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- DSML 变体在围栏内不触发 ----
|
||||
|
||||
func TestSieve_DSMLInsideFenceIgnored(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"```\n",
|
||||
"<|DSML|tool_calls>\n",
|
||||
`<|DSML|invoke name="read_file">`,
|
||||
`<|DSML|parameter name="path">x</|DSML|parameter>`,
|
||||
"</|DSML|invoke>\n",
|
||||
"</|DSML|tool_calls>\n",
|
||||
"```\n",
|
||||
"结束",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 0 {
|
||||
t.Fatalf("围栏内的 DSML 工具调用不应触发，got %d", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 工具调用前后有丰富文本 ----
|
||||
|
||||
func TestSieve_RichTextAroundToolCall(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"我来帮你查看文件内容。\n\n",
|
||||
"首先读取 README：\n",
|
||||
`<tool_calls><invoke name="read_file"><parameter name="path">README.md</parameter></invoke></tool_calls>`,
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 1 {
|
||||
t.Fatalf("应有一个工具调用，got %d", tc)
|
||||
}
|
||||
if !strings.Contains(text.String(), "帮你查看") {
|
||||
t.Fatalf("前置文本丢失, got %q", text.String())
|
||||
}
|
||||
if strings.Contains(text.String(), "<invoke") {
|
||||
t.Fatalf("工具标签泄漏, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 工具调用在 CDATA 包含代码围栏 ----
|
||||
|
||||
func TestSieve_ToolCallWithCDATAContainingFence(t *testing.T) {
|
||||
var state State
|
||||
payload := "```python\nprint('hello')\n```"
|
||||
chunks := []string{
|
||||
"<tool_calls>\n",
|
||||
`<invoke name="write_file">` + "\n",
|
||||
`<parameter name="path">test.md</parameter>` + "\n",
|
||||
`<parameter name="content"><![CDATA[` + payload + `]]></parameter>` + "\n",
|
||||
"</invoke>\n",
|
||||
"</tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"write_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"write_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
var gotContent any
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
if len(e.ToolCalls) > 0 {
|
||||
tc += len(e.ToolCalls)
|
||||
gotContent = e.ToolCalls[0].Input["content"]
|
||||
}
|
||||
}
|
||||
if tc != 1 {
|
||||
t.Fatalf("应有一个工具调用，got %d", tc)
|
||||
}
|
||||
content, _ := gotContent.(string)
|
||||
if content != payload {
|
||||
t.Fatalf("CDATA 内围栏内容应完整保留，got %q want %q", content, payload)
|
||||
}
|
||||
if text.Len() != 0 {
|
||||
t.Fatalf("不应有文本泄漏, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 极端 token 拆分 ----
|
||||
|
||||
// 工具标签被拆成单字符流式到达
|
||||
func TestSieve_CharByCharToolCall(t *testing.T) {
|
||||
var state State
|
||||
full := `<tool_calls><invoke name="read_file"><parameter name="path">go.mod</parameter></invoke></tool_calls>`
|
||||
var events []Event
|
||||
for _, ch := range full {
|
||||
events = append(events, ProcessChunk(&state, string(ch), []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 1 {
|
||||
t.Fatalf("单字符流式应解析出工具调用，got %d", tc)
|
||||
}
|
||||
if strings.Contains(text.String(), "invoke") {
|
||||
t.Fatalf("标签泄漏, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 混合格式变体 ----
|
||||
|
||||
// 全宽竖线 wrapper + DSML invoke
|
||||
func TestSieve_FullwidthPipeWrapperDSMLInvoke(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"<｜tool_calls>\n",
|
||||
"<|DSML|invoke name=\"read_file\">\n",
|
||||
"<|DSML|parameter name=\"path\">README.md</|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"</｜tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
if tc != 1 {
|
||||
t.Fatalf("全宽+DSML混合应解析成功，got %d", tc)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(text.String()), "dsml") {
|
||||
t.Fatalf("DSML 标签泄漏, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 未闭合工具块应回退为文本 ----
|
||||
|
||||
func TestSieve_UnclosedToolCallBlockFallsBack(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"<tool_calls>\n",
|
||||
`<invoke name="read_file">` + "\n",
|
||||
`<parameter name="path">README.md</parameter>` + "\n",
|
||||
// 缺少 </invoke> 和 </tool_calls>
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
tc += len(e.ToolCalls)
|
||||
}
|
||||
// 未闭合的应回退为文本，不应丢失
|
||||
if text.String() == "" {
|
||||
t.Fatalf("未闭合工具块不应丢失所有内容")
|
||||
}
|
||||
if tc != 0 {
|
||||
t.Fatalf("未闭合工具块不应解析出工具调用，got %d", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 文本中 mention 标签变体名 + 真正的工具调用 ----
|
||||
|
||||
// 模型输出 commit message 文本中包含 <dsml|tool_calls> 等 mention，
|
||||
// 紧随其后是真正的 DSML 工具调用。mention 的变体和实际工具调用变体不同。
|
||||
func TestSieve_TagMentionInTextThenRealToolCall(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"建议的 commit message：\n\nfeat: expand DSML alias support\n\n",
|
||||
"Add support for <dsml|tool_calls>, ",
|
||||
"<｜tool_calls> (fullwidth pipe),\n",
|
||||
"and <|tool_calls> wrapper variants.\n\n",
|
||||
"<|DSML|tool_calls>\n",
|
||||
"<|DSML|invoke name=\"Bash\">\n",
|
||||
"<|DSML|parameter name=\"command\"><![CDATA[git status]]></|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"</|DSML|tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"Bash"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"Bash"})...)
|
||||
|
||||
var text strings.Builder
|
||||
tc := 0
|
||||
var names []string
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
for _, call := range e.ToolCalls {
|
||||
tc++
|
||||
names = append(names, call.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if tc != 1 {
|
||||
t.Fatalf("应解析出 1 个工具调用，got %d, text=%q", tc, text.String())
|
||||
}
|
||||
if names[0] != "Bash" {
|
||||
t.Fatalf("应解析出 Bash，got %v", names)
|
||||
}
|
||||
if !strings.Contains(text.String(), "commit message") {
|
||||
t.Fatalf("前置文本应保留, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSieve_SameVariantTagMentionInTextThenRealToolCall(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"Summary: support canonical <tool_calls> and DSML <|DSML|tool_calls> wrappers.\n\n",
|
||||
"<|DSML|tool_calls>\n",
|
||||
"<|DSML|invoke name=\"Bash\">\n",
|
||||
"<|DSML|parameter name=\"command\"><![CDATA[git status]]></|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"</|DSML|tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"Bash"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"Bash"})...)
|
||||
|
||||
var text strings.Builder
|
||||
var callName string
|
||||
var command string
|
||||
callCount := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
for _, call := range e.ToolCalls {
|
||||
callCount++
|
||||
callName = call.Name
|
||||
command, _ = call.Input["command"].(string)
|
||||
}
|
||||
}
|
||||
|
||||
if callCount != 1 {
|
||||
t.Fatalf("应解析出 1 个工具调用，got %d, text=%q", callCount, text.String())
|
||||
}
|
||||
if callName != "Bash" {
|
||||
t.Fatalf("应解析出 Bash，got %q", callName)
|
||||
}
|
||||
if command != "git status" {
|
||||
t.Fatalf("应解析出 command，got %q", command)
|
||||
}
|
||||
if !strings.Contains(text.String(), "Summary:") {
|
||||
t.Fatalf("前置文本应保留, got %q", text.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSieve_ReviewSampleWithAliasMentionsPreservesBodyAndToolCalls(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"Done reviewing the diff. Here's my analysis before we commit:\n\n",
|
||||
"Summary of Changes\n",
|
||||
"DSML wrapper variant support — recognize aliases (<dsml|tool_calls>, <|tool_calls>, <｜tool_calls>) alongside canonical <tool_calls> and <|DSML|tool_calls> wrappers.\n\n",
|
||||
"<|DSML|tool_calls>\n",
|
||||
"<|DSML|invoke name=\"Bash\">\n",
|
||||
"<|DSML|parameter name=\"command\"><![CDATA[git add docs/toolcall-semantics.md internal/toolstream/tool_sieve_xml.go]]></|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"description\"><![CDATA[Stage all relevant changed files]]></|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"<|DSML|invoke name=\"Bash\">\n",
|
||||
"<|DSML|parameter name=\"command\"><![CDATA[git commit -m \"$(cat <<'EOF'\nfeat(toolstream): expand DSML wrapper detection\n\nSupport DSML wrapper aliases: <dsml|tool_calls>, <|tool_calls>, <｜tool_calls> alongside existing canonical wrappers.\nEOF\n)\"]]></|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"description\"><![CDATA[Create commit with all staged changes]]></|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"</|DSML|tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"Bash"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"Bash"})...)
|
||||
|
||||
var text strings.Builder
|
||||
var commands []string
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
for _, call := range e.ToolCalls {
|
||||
if call.Name == "Bash" {
|
||||
cmd, _ := call.Input["command"].(string)
|
||||
commands = append(commands, cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(commands) != 2 {
|
||||
t.Fatalf("应解析出 2 个 Bash 工具调用，got %d, text=%q", len(commands), text.String())
|
||||
}
|
||||
if !strings.Contains(text.String(), "<|DSML|tool_calls> wrappers") {
|
||||
t.Fatalf("正文中的 DSML mention 应保留, got %q", text.String())
|
||||
}
|
||||
if !strings.Contains(text.String(), "Summary of Changes") {
|
||||
t.Fatalf("前置正文应完整保留, got %q", text.String())
|
||||
}
|
||||
if strings.Contains(text.String(), "git add docs/toolcall-semantics.md") {
|
||||
t.Fatalf("真实工具参数不应泄漏到正文, got %q", text.String())
|
||||
}
|
||||
if !strings.Contains(commands[0], "git add") || !strings.Contains(commands[1], "git commit") {
|
||||
t.Fatalf("工具参数解析不符合预期, got %#v", commands)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSieve_ChineseReviewSamplePreservesInlineDSMLMention(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"# Context from my IDE setup:\n\n## My request for Codex:\n",
|
||||
"基于我的审查，这是工作区更改的总结和提交。\n\n## 审查报告\n\n### 文档\n\nAPI.md 中的工具调用部分缺少针对新 DSML 别名的更新——它只提到了 `",
|
||||
"<|DSML|tool_calls>` 和 canonical `<tool_calls>`。由于这涉及 API 兼容性和文档准确性，需要在下游进行记录。\n\n",
|
||||
"### 代码\n\n所有更改现在一致地处理四个 DSML wrapper 变体。\n\n现在提交已暂存的更改。\n\n",
|
||||
"<|DSML|tool_calls>\n",
|
||||
" <|DSML|invoke name=\"Bash\">\n",
|
||||
" <|DSML|parameter name=\"command\"><![CDATA[git commit -m \"$(cat <<'EOF'\nfeat: expand DSML tool-call alias and fence handling\nEOF\n)\"]]></|DSML|parameter>\n",
|
||||
" <|DSML|parameter name=\"description\"><![CDATA[Commit staged changes]]></|DSML|parameter>\n",
|
||||
" </|DSML|invoke>\n",
|
||||
"</|DSML|tool_calls>\n\n补充",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"Bash"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"Bash"})...)
|
||||
|
||||
var text strings.Builder
|
||||
callCount := 0
|
||||
for _, e := range events {
|
||||
text.WriteString(e.Content)
|
||||
callCount += len(e.ToolCalls)
|
||||
}
|
||||
|
||||
if callCount != 1 {
|
||||
t.Fatalf("应解析出 1 个工具调用，got %d, text=%q", callCount, text.String())
|
||||
}
|
||||
want := "它只提到了 `<|DSML|tool_calls>` 和 canonical `<tool_calls>`。由于这涉及 API 兼容性"
|
||||
if !strings.Contains(text.String(), want) {
|
||||
t.Fatalf("正文不应在 inline DSML mention 处截断, want contains %q, got %q", want, text.String())
|
||||
}
|
||||
if !strings.Contains(text.String(), "补充") {
|
||||
t.Fatalf("工具块后的正文应保留, got %q", text.String())
|
||||
}
|
||||
if strings.Contains(text.String(), "<|DSML|invoke") {
|
||||
t.Fatalf("真实工具块不应泄漏到正文, got %q", text.String())
|
||||
}
|
||||
}
|
||||
59
internal/toolstream/fence_edge_sieve_test.go
Normal file
59
internal/toolstream/fence_edge_sieve_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package toolstream
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 波浪线围栏内的工具调用标签不应触发工具调用
|
||||
func TestProcessToolSieveTildeFenceDoesNotTriggerToolCall(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"示例：\n~~~xml\n",
|
||||
"<tool_calls><invoke name=\"read_file\"><parameter name=\"path\">README.md</parameter></invoke></tool_calls>\n",
|
||||
"~~~\n",
|
||||
"完毕。",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var textContent strings.Builder
|
||||
toolCalls := 0
|
||||
for _, evt := range events {
|
||||
textContent.WriteString(evt.Content)
|
||||
toolCalls += len(evt.ToolCalls)
|
||||
}
|
||||
|
||||
if toolCalls != 0 {
|
||||
t.Fatalf("expected tilde-fenced tool example to stay text, got %d tool calls", toolCalls)
|
||||
}
|
||||
if !strings.Contains(textContent.String(), "示例") || !strings.Contains(textContent.String(), "完毕") {
|
||||
t.Fatalf("expected surrounding text preserved, got %q", textContent.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 4 反引号嵌套 3 反引号（内含工具标签）不应触发
|
||||
func TestProcessToolSieveNestedFourBacktickFenceDoesNotTrigger(t *testing.T) {
|
||||
var state State
|
||||
input := "说明：\n````xml\n```\n<tool_calls><invoke name=\"read_file\"><parameter name=\"path\">x</parameter></invoke></tool_calls>\n```\n````\n结束。"
|
||||
chunks := strings.SplitAfter(input, "\n")
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"read_file"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"read_file"})...)
|
||||
|
||||
var textContent strings.Builder
|
||||
toolCalls := 0
|
||||
for _, evt := range events {
|
||||
textContent.WriteString(evt.Content)
|
||||
toolCalls += len(evt.ToolCalls)
|
||||
}
|
||||
|
||||
if toolCalls != 0 {
|
||||
t.Fatalf("expected 4-backtick fenced example to stay text, got %d tool calls", toolCalls)
|
||||
}
|
||||
}
|
||||
@@ -6,21 +6,22 @@ import (
|
||||
)
|
||||
|
||||
type State struct {
|
||||
pending strings.Builder
|
||||
capture strings.Builder
|
||||
capturing bool
|
||||
codeFenceStack []int
|
||||
codeFencePendingTicks int
|
||||
codeFenceLineStart bool
|
||||
pendingToolRaw string
|
||||
pendingToolCalls []toolcall.ParsedToolCall
|
||||
disableDeltas bool
|
||||
toolNameSent bool
|
||||
toolName string
|
||||
toolArgsStart int
|
||||
toolArgsSent int
|
||||
toolArgsString bool
|
||||
toolArgsDone bool
|
||||
pending strings.Builder
|
||||
capture strings.Builder
|
||||
capturing bool
|
||||
codeFenceStack []int
|
||||
codeFencePendingTicks int
|
||||
codeFencePendingTildes int
|
||||
codeFenceNotLineStart bool // inverted: zero-value false means "at line start"
|
||||
pendingToolRaw string
|
||||
pendingToolCalls []toolcall.ParsedToolCall
|
||||
disableDeltas bool
|
||||
toolNameSent bool
|
||||
toolName string
|
||||
toolArgsStart int
|
||||
toolArgsSent int
|
||||
toolArgsString bool
|
||||
toolArgsDone bool
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
@@ -63,7 +64,8 @@ func insideCodeFenceWithState(state *State, text string) bool {
|
||||
simulated := simulateCodeFenceState(
|
||||
state.codeFenceStack,
|
||||
state.codeFencePendingTicks,
|
||||
state.codeFenceLineStart,
|
||||
state.codeFencePendingTildes,
|
||||
!state.codeFenceNotLineStart,
|
||||
text,
|
||||
)
|
||||
return len(simulated.stack) > 0
|
||||
@@ -73,7 +75,7 @@ func insideCodeFence(text string) bool {
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
return len(simulateCodeFenceState(nil, 0, true, text).stack) > 0
|
||||
return len(simulateCodeFenceState(nil, 0, 0, true, text).stack) > 0
|
||||
}
|
||||
|
||||
func updateCodeFenceState(state *State, text string) {
|
||||
@@ -83,43 +85,65 @@ func updateCodeFenceState(state *State, text string) {
|
||||
next := simulateCodeFenceState(
|
||||
state.codeFenceStack,
|
||||
state.codeFencePendingTicks,
|
||||
state.codeFenceLineStart,
|
||||
state.codeFencePendingTildes,
|
||||
!state.codeFenceNotLineStart,
|
||||
text,
|
||||
)
|
||||
state.codeFenceStack = next.stack
|
||||
state.codeFencePendingTicks = next.pendingTicks
|
||||
state.codeFenceLineStart = next.lineStart
|
||||
state.codeFencePendingTildes = next.pendingTildes
|
||||
state.codeFenceNotLineStart = !next.lineStart
|
||||
}
|
||||
|
||||
type codeFenceSimulation struct {
|
||||
stack []int
|
||||
pendingTicks int
|
||||
lineStart bool
|
||||
stack []int
|
||||
pendingTicks int
|
||||
pendingTildes int
|
||||
lineStart bool
|
||||
}
|
||||
|
||||
func simulateCodeFenceState(stack []int, pendingTicks int, lineStart bool, text string) codeFenceSimulation {
|
||||
func simulateCodeFenceState(stack []int, pendingTicks, pendingTildes int, lineStart bool, text string) codeFenceSimulation {
|
||||
chunk := text
|
||||
nextStack := append([]int(nil), stack...)
|
||||
ticks := pendingTicks
|
||||
tildes := pendingTildes
|
||||
atLineStart := lineStart
|
||||
|
||||
flushTicks := func() {
|
||||
flushPending := func() {
|
||||
if ticks > 0 {
|
||||
if atLineStart && ticks >= 3 {
|
||||
applyFenceMarker(&nextStack, ticks)
|
||||
applyFenceMarker(&nextStack, ticks) // positive = backtick
|
||||
}
|
||||
atLineStart = false
|
||||
ticks = 0
|
||||
}
|
||||
if tildes > 0 {
|
||||
if atLineStart && tildes >= 3 {
|
||||
applyFenceMarker(&nextStack, -tildes) // negative = tilde
|
||||
}
|
||||
atLineStart = false
|
||||
tildes = 0
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(chunk); i++ {
|
||||
ch := chunk[i]
|
||||
if ch == '`' {
|
||||
if tildes > 0 {
|
||||
// Mixed chars — flush tildes first.
|
||||
flushPending()
|
||||
}
|
||||
ticks++
|
||||
continue
|
||||
}
|
||||
flushTicks()
|
||||
if ch == '~' {
|
||||
if ticks > 0 {
|
||||
flushPending()
|
||||
}
|
||||
tildes++
|
||||
continue
|
||||
}
|
||||
flushPending()
|
||||
switch ch {
|
||||
case '\n', '\r':
|
||||
atLineStart = true
|
||||
@@ -134,24 +158,43 @@ func simulateCodeFenceState(stack []int, pendingTicks int, lineStart bool, text
|
||||
}
|
||||
|
||||
return codeFenceSimulation{
|
||||
stack: nextStack,
|
||||
pendingTicks: ticks,
|
||||
lineStart: atLineStart,
|
||||
stack: nextStack,
|
||||
pendingTicks: ticks,
|
||||
pendingTildes: tildes,
|
||||
lineStart: atLineStart,
|
||||
}
|
||||
}
|
||||
|
||||
func applyFenceMarker(stack *[]int, ticks int) {
|
||||
if stack == nil || ticks <= 0 {
|
||||
// applyFenceMarker pushes or pops a fence marker on the stack.
|
||||
// Positive values represent backtick fences, negative represent tilde fences.
|
||||
// A closing marker must match the sign (type) of the opening marker.
|
||||
func applyFenceMarker(stack *[]int, marker int) {
|
||||
if stack == nil || marker == 0 {
|
||||
return
|
||||
}
|
||||
if len(*stack) == 0 {
|
||||
*stack = append(*stack, ticks)
|
||||
*stack = append(*stack, marker)
|
||||
return
|
||||
}
|
||||
top := (*stack)[len(*stack)-1]
|
||||
if ticks >= top {
|
||||
// Signs must match: backtick closes backtick, tilde closes tilde.
|
||||
sameType := (top > 0 && marker > 0) || (top < 0 && marker < 0)
|
||||
if !sameType {
|
||||
// Different fence type — treat as nested.
|
||||
*stack = append(*stack, marker)
|
||||
return
|
||||
}
|
||||
absMarker := marker
|
||||
absTop := top
|
||||
if absMarker < 0 {
|
||||
absMarker = -absMarker
|
||||
}
|
||||
if absTop < 0 {
|
||||
absTop = -absTop
|
||||
}
|
||||
if absMarker >= absTop {
|
||||
*stack = (*stack)[:len(*stack)-1]
|
||||
return
|
||||
}
|
||||
*stack = append(*stack, ticks)
|
||||
*stack = append(*stack, marker)
|
||||
}
|
||||
|
||||
@@ -9,13 +9,22 @@ import (
|
||||
// --- XML tool call support for the streaming sieve ---
|
||||
|
||||
//nolint:unused // kept as explicit tag inventory for future XML sieve refinements.
|
||||
var xmlToolCallClosingTags = []string{"</tool_calls>", "</|dsml|tool_calls>"}
|
||||
var xmlToolCallOpeningTags = []string{"<tool_calls", "<invoke", "<|dsml|tool_calls", "<|dsml|invoke"}
|
||||
var xmlToolCallClosingTags = []string{"</tool_calls>", "</|dsml|tool_calls>", "</dsml|tool_calls>", "</｜tool_calls>", "</|tool_calls>"}
|
||||
var xmlToolCallOpeningTags = []string{
|
||||
"<tool_calls", "<invoke",
|
||||
"<|dsml|tool_calls", "<|dsml|invoke",
|
||||
"<dsml|tool_calls", "<dsml|invoke",
|
||||
"<｜tool_calls", "<｜invoke",
|
||||
"<|tool_calls", "<|invoke",
|
||||
}
|
||||
|
||||
// xmlToolCallTagPairs maps each opening tag to its expected closing tag.
|
||||
// Order matters: longer/wrapper tags must be checked first.
|
||||
var xmlToolCallTagPairs = []struct{ open, close string }{
|
||||
{"<|dsml|tool_calls", "</|dsml|tool_calls>"},
|
||||
{"<dsml|tool_calls", "</dsml|tool_calls>"},
|
||||
{"<｜tool_calls", "</｜tool_calls>"},
|
||||
{"<|tool_calls", "</|tool_calls>"},
|
||||
{"<tool_calls", "</tool_calls>"},
|
||||
}
|
||||
|
||||
@@ -28,38 +37,80 @@ var xmlToolCallBlockPattern = regexp.MustCompile(`(?is)((?:<tool_calls\b|<\|dsml
|
||||
var xmlToolTagsToDetect = []string{
|
||||
"<|dsml|tool_calls>", "<|dsml|tool_calls\n", "<|dsml|tool_calls ",
|
||||
"<|dsml|invoke ", "<|dsml|invoke\n", "<|dsml|invoke\t", "<|dsml|invoke\r",
|
||||
"<dsml|tool_calls>", "<dsml|tool_calls\n", "<dsml|tool_calls ",
|
||||
"<dsml|invoke ", "<dsml|invoke\n", "<dsml|invoke\t", "<dsml|invoke\r",
|
||||
"<｜tool_calls>", "<｜tool_calls\n", "<｜tool_calls ",
|
||||
"<｜invoke ", "<｜invoke\n", "<｜invoke\t", "<｜invoke\r",
|
||||
"<|tool_calls>", "<|tool_calls\n", "<|tool_calls ",
|
||||
"<|invoke ", "<|invoke\n", "<|invoke\t", "<|invoke\r",
|
||||
"<tool_calls>", "<tool_calls\n", "<tool_calls ", "<invoke ", "<invoke\n", "<invoke\t", "<invoke\r",
|
||||
}
|
||||
|
||||
// consumeXMLToolCapture tries to extract complete XML tool call blocks from captured text.
|
||||
func consumeXMLToolCapture(captured string, toolNames []string) (prefix string, calls []toolcall.ParsedToolCall, suffix string, ready bool) {
|
||||
lower := strings.ToLower(captured)
|
||||
// Find the FIRST matching open/close pair for the canonical wrapper.
|
||||
for _, pair := range xmlToolCallTagPairs {
|
||||
openIdx := strings.Index(lower, pair.open)
|
||||
if openIdx < 0 {
|
||||
continue
|
||||
}
|
||||
// Find the matching closing tag outside CDATA. Long write-file tool
|
||||
// calls often contain XML examples in CDATA, including </tool_calls>.
|
||||
closeIdx := findXMLCloseOutsideCDATA(captured, pair.close, openIdx+len(pair.open))
|
||||
if closeIdx < 0 {
|
||||
// Opening tag is present but its specific closing tag hasn't arrived.
|
||||
// Return not-ready so we keep buffering until the canonical wrapper closes.
|
||||
return "", nil, "", false
|
||||
}
|
||||
closeEnd := closeIdx + len(pair.close)
|
||||
anyOpenFound := false
|
||||
type candidate struct {
|
||||
start int
|
||||
prefix string
|
||||
calls []toolcall.ParsedToolCall
|
||||
suffix string
|
||||
}
|
||||
type rejectedBlock struct {
|
||||
start int
|
||||
prefix string
|
||||
suffix string
|
||||
}
|
||||
var best *candidate
|
||||
var rejected *rejectedBlock
|
||||
|
||||
xmlBlock := captured[openIdx:closeEnd]
|
||||
prefixPart := captured[:openIdx]
|
||||
suffixPart := captured[closeEnd:]
|
||||
parsed := toolcall.ParseToolCalls(xmlBlock, toolNames)
|
||||
if len(parsed) > 0 {
|
||||
prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart)
|
||||
return prefixPart, parsed, suffixPart, true
|
||||
// Scan every wrapper occurrence. Prose can mention a wrapper tag before the
|
||||
// actual tool block, including the same variant as the real block.
|
||||
for _, pair := range xmlToolCallTagPairs {
|
||||
searchFrom := 0
|
||||
for searchFrom < len(lower) {
|
||||
openIdx := findXMLOpenOutsideCDATA(captured, pair.open, searchFrom)
|
||||
if openIdx < 0 {
|
||||
break
|
||||
}
|
||||
// Find the matching closing tag outside CDATA. Long write-file tool
|
||||
// calls often contain XML examples in CDATA, including </tool_calls>.
|
||||
closeIdx := findMatchingXMLToolWrapperClose(captured, pair.open, pair.close, openIdx)
|
||||
if closeIdx < 0 {
|
||||
anyOpenFound = true
|
||||
searchFrom = openIdx + len(pair.open)
|
||||
continue
|
||||
}
|
||||
closeEnd := closeIdx + len(pair.close)
|
||||
|
||||
xmlBlock := captured[openIdx:closeEnd]
|
||||
prefixPart := captured[:openIdx]
|
||||
suffixPart := captured[closeEnd:]
|
||||
parsed := toolcall.ParseToolCalls(xmlBlock, toolNames)
|
||||
if len(parsed) > 0 {
|
||||
prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart)
|
||||
if best == nil || openIdx < best.start {
|
||||
best = &candidate{start: openIdx, prefix: prefixPart, calls: parsed, suffix: suffixPart}
|
||||
}
|
||||
break
|
||||
}
|
||||
if rejected == nil || openIdx < rejected.start {
|
||||
rejected = &rejectedBlock{start: openIdx, prefix: prefixPart + xmlBlock, suffix: suffixPart}
|
||||
}
|
||||
searchFrom = openIdx + len(pair.open)
|
||||
}
|
||||
}
|
||||
if best != nil {
|
||||
return best.prefix, best.calls, best.suffix, true
|
||||
}
|
||||
if anyOpenFound {
|
||||
// At least one opening tag was found but none had a matching close tag.
|
||||
// Keep buffering until a closing tag arrives.
|
||||
return "", nil, "", false
|
||||
}
|
||||
if rejected != nil {
|
||||
// If this block failed to become a tool call, pass it through as text.
|
||||
return prefixPart + xmlBlock, nil, suffixPart, true
|
||||
return rejected.prefix, nil, rejected.suffix, true
|
||||
}
|
||||
if !containsAnyToolCallWrapper(lower) {
|
||||
invokeIdx, dsml := firstInvokeIndex(lower)
|
||||
@@ -86,6 +137,88 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
|
||||
return "", nil, "", false
|
||||
}
|
||||
|
||||
func findMatchingXMLToolWrapperClose(s, openTag, closeTag string, openIdx int) int {
|
||||
if s == "" || openTag == "" || closeTag == "" || openIdx < 0 {
|
||||
return -1
|
||||
}
|
||||
lower := strings.ToLower(s)
|
||||
openTarget := strings.ToLower(openTag)
|
||||
closeTarget := strings.ToLower(closeTag)
|
||||
depth := 1
|
||||
for i := openIdx + len(openTarget); i < len(s); {
|
||||
switch {
|
||||
case strings.HasPrefix(lower[i:], "<![cdata["):
|
||||
end := strings.Index(lower[i+len("<![cdata["):], "]]>")
|
||||
if end < 0 {
|
||||
return -1
|
||||
}
|
||||
i += len("<![cdata[") + end + len("]]>")
|
||||
case strings.HasPrefix(lower[i:], "<!--"):
|
||||
end := strings.Index(lower[i+len("<!--"):], "-->")
|
||||
if end < 0 {
|
||||
return -1
|
||||
}
|
||||
i += len("<!--") + end + len("-->")
|
||||
case strings.HasPrefix(lower[i:], closeTarget):
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i
|
||||
}
|
||||
i += len(closeTarget)
|
||||
case strings.HasPrefix(lower[i:], openTarget) && hasXMLToolTagBoundary(s, i+len(openTarget)):
|
||||
depth++
|
||||
i += len(openTarget)
|
||||
default:
|
||||
i++
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func findXMLOpenOutsideCDATA(s, openTag string, start int) int {
|
||||
if s == "" || openTag == "" {
|
||||
return -1
|
||||
}
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
lower := strings.ToLower(s)
|
||||
target := strings.ToLower(openTag)
|
||||
for i := start; i < len(s); {
|
||||
switch {
|
||||
case strings.HasPrefix(lower[i:], "<![cdata["):
|
||||
end := strings.Index(lower[i+len("<![cdata["):], "]]>")
|
||||
if end < 0 {
|
||||
return -1
|
||||
}
|
||||
i += len("<![cdata[") + end + len("]]>")
|
||||
case strings.HasPrefix(lower[i:], "<!--"):
|
||||
end := strings.Index(lower[i+len("<!--"):], "-->")
|
||||
if end < 0 {
|
||||
return -1
|
||||
}
|
||||
i += len("<!--") + end + len("-->")
|
||||
case strings.HasPrefix(lower[i:], target) && hasXMLToolTagBoundary(s, i+len(target)):
|
||||
return i
|
||||
default:
|
||||
i++
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func hasXMLToolTagBoundary(text string, idx int) bool {
|
||||
if idx >= len(text) {
|
||||
return true
|
||||
}
|
||||
switch text[idx] {
|
||||
case ' ', '\t', '\n', '\r', '>', '/':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// hasOpenXMLToolTag returns true if captured text contains an XML tool opening tag
|
||||
// whose SPECIFIC closing tag has not appeared yet.
|
||||
func hasOpenXMLToolTag(captured string) bool {
|
||||
@@ -144,12 +277,24 @@ func shouldKeepBareInvokeCapture(captured string) bool {
|
||||
}
|
||||
|
||||
func containsAnyToolCallWrapper(lower string) bool {
|
||||
return strings.Contains(lower, "<tool_calls") || strings.Contains(lower, "<|dsml|tool_calls")
|
||||
return strings.Contains(lower, "<tool_calls") ||
|
||||
strings.Contains(lower, "<|dsml|tool_calls") ||
|
||||
strings.Contains(lower, "<dsml|tool_calls") ||
|
||||
strings.Contains(lower, "<｜tool_calls") ||
|
||||
strings.Contains(lower, "<|tool_calls")
|
||||
}
|
||||
|
||||
func firstInvokeIndex(lower string) (int, bool) {
|
||||
xmlIdx := strings.Index(lower, "<invoke")
|
||||
dsmlIdx := strings.Index(lower, "<|dsml|invoke")
|
||||
// Check all DSML-like invoke prefixes.
|
||||
dsmlPrefixes := []string{"<|dsml|invoke", "<dsml|invoke", "<｜invoke", "<|invoke"}
|
||||
dsmlIdx := -1
|
||||
for _, prefix := range dsmlPrefixes {
|
||||
idx := strings.Index(lower, prefix)
|
||||
if idx >= 0 && (dsmlIdx < 0 || idx < dsmlIdx) {
|
||||
dsmlIdx = idx
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case xmlIdx < 0:
|
||||
return dsmlIdx, dsmlIdx >= 0
|
||||
|
||||
@@ -678,3 +678,96 @@ func TestProcessToolSieveRepairsMissingOpeningWrapperWithoutLeakingInvokeText(t
|
||||
t.Fatalf("expected repaired missing-wrapper stream not to leak xml text, got %q", textContent.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Test fullwidth pipe variant: <｜tool_calls> (U+FF5C) should be buffered and parsed.
|
||||
func TestProcessToolSieveFullwidthPipeVariantDoesNotLeak(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"<\uff5ctool_calls>\n",
|
||||
"<invoke name=\"execute_command\">\n",
|
||||
"<parameter name=\"command\">git status</parameter>\n",
|
||||
"</invoke>\n",
|
||||
"</\uff5ctool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"execute_command"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"execute_command"})...)
|
||||
|
||||
var textContent string
|
||||
var toolCalls int
|
||||
for _, evt := range events {
|
||||
textContent += evt.Content
|
||||
toolCalls += len(evt.ToolCalls)
|
||||
}
|
||||
|
||||
if strings.Contains(textContent, "invoke") || strings.Contains(textContent, "execute_command") {
|
||||
t.Fatalf("fullwidth pipe variant leaked to text: %q", textContent)
|
||||
}
|
||||
if toolCalls != 1 {
|
||||
t.Fatalf("expected one tool call from fullwidth pipe variant, got %d events=%#v", toolCalls, events)
|
||||
}
|
||||
}
|
||||
|
||||
// Test <DSML|tool_calls> with <|DSML|invoke> (DSML prefix without leading pipe on wrapper).
|
||||
func TestProcessToolSieveDSMLPrefixVariantDoesNotLeak(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"<DSML|tool_calls>\n",
|
||||
" <|DSML|invoke name=\"execute_command\">\n",
|
||||
" <|DSML|parameter name=\"command\"><![CDATA[git status]]></|DSML|parameter>\n",
|
||||
" </|DSML|invoke>\n",
|
||||
"</DSML|tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"execute_command"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"execute_command"})...)
|
||||
|
||||
var textContent string
|
||||
var toolCalls int
|
||||
for _, evt := range events {
|
||||
textContent += evt.Content
|
||||
toolCalls += len(evt.ToolCalls)
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(textContent), "dsml") || strings.Contains(textContent, "execute_command") {
|
||||
t.Fatalf("DSML prefix variant leaked to text: %q", textContent)
|
||||
}
|
||||
if toolCalls != 1 {
|
||||
t.Fatalf("expected one tool call from DSML prefix variant, got %d events=%#v", toolCalls, events)
|
||||
}
|
||||
}
|
||||
|
||||
// Test <DSML|tool_calls> with <DSML|invoke> (no pipe anywhere) should be buffered and parsed.
|
||||
func TestProcessToolSieveDSMLBarePrefixVariantDoesNotLeak(t *testing.T) {
|
||||
var state State
|
||||
chunks := []string{
|
||||
"<DSML|tool_calls>\n",
|
||||
"<DSML|invoke name=\"execute_command\">\n",
|
||||
"<DSML|parameter name=\"command\"><![CDATA[git status]]></DSML|parameter>\n",
|
||||
"</DSML|invoke>\n",
|
||||
"</DSML|tool_calls>",
|
||||
}
|
||||
var events []Event
|
||||
for _, c := range chunks {
|
||||
events = append(events, ProcessChunk(&state, c, []string{"execute_command"})...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"execute_command"})...)
|
||||
|
||||
var textContent string
|
||||
var toolCalls int
|
||||
for _, evt := range events {
|
||||
textContent += evt.Content
|
||||
toolCalls += len(evt.ToolCalls)
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(textContent), "dsml") || strings.Contains(textContent, "execute_command") {
|
||||
t.Fatalf("DSML bare prefix variant leaked to text: %q", textContent)
|
||||
}
|
||||
if toolCalls != 1 {
|
||||
t.Fatalf("expected one tool call from DSML bare prefix variant, got %d events=%#v", toolCalls, events)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user