refactor: unify Go/Node XML tool markup scanning and expand DSML alias support

- Add shared ToolMarkupTag scanner (toolcalls_scan.go) replacing hardcoded alias tables
- Support DSML collapsed tag names (<DSMLtool_calls>, <DSMLinvoke>, <DSMLparameter>)
- Parse JSON literal values from parameter bodies (123→number, true→bool, null)
- Recover unclosed CDATA in final parse/flush via SanitizeLooseCDATA
- Align Go and Node implementations (scanToolMarkupTagAt, findMatchingToolMarkupClose)
- Reject bare <invoke> as unsupported syntax, only tool_calls wrapper triggers tool path
- Update API.md and toolcall-semantics.md documentation

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

internal/toolcall/regression_test.go

@@ -12,9 +12,9 @@ func TestRegression_RobustXMLAndCDATA(t *testing.T) {
 		expected []ParsedToolCall
 	}{
 		{
-			name:     "Standard JSON parameters (Regression)",
+			name:     "Standard JSON scalar parameters (Regression)",
 			text:     `<tool_calls><invoke name="foo"><parameter name="a">1</parameter></invoke></tool_calls>`,
-			expected: []ParsedToolCall{{Name: "foo", Input: map[string]any{"a": "1"}}},
+			expected: []ParsedToolCall{{Name: "foo", Input: map[string]any{"a": float64(1)}}},
 		},
 		{
 			name:     "XML tags parameters (Regression)",

internal/toolcall/toolcalls_dsml.go

@@ -6,96 +6,17 @@ func normalizeDSMLToolCallMarkup(text string) (string, bool) {
 	if text == "" {
 		return "", true
 	}
-	hasAliasLikeMarkup, _ := toolMarkupStylesOutsideIgnored(text)
+	hasAliasLikeMarkup, _ := ContainsToolMarkupSyntaxOutsideIgnored(text)
 	if !hasAliasLikeMarkup {
 		return text, true
 	}
-	// Always normalize DSML aliases to canonical form, even when canonical
-	// tags coexist. Models frequently mix DSML wrapper tags with canonical
-	// inner tags (e.g., <｜tool_calls><invoke name="...">).
-	return replaceDSMLToolMarkupOutsideIgnored(text), true
+	return rewriteDSMLToolMarkupOutsideIgnored(text), true
 }
 
-var dsmlToolMarkupAliases = []struct {
-	from string
-	to   string
-}{
-	{"<|dsml|tool_calls", "<tool_calls"},
-	{"</|dsml|tool_calls>", "</tool_calls>"},
-	{"<|dsml|invoke", "<invoke"},
-	{"</|dsml|invoke>", "</invoke>"},
-	{"<|dsml|parameter", "<parameter"},
-	{"</|dsml|parameter>", "</parameter>"},
-	{"<|dsml tool_calls", "<tool_calls"},
-	{"</|dsml tool_calls>", "</tool_calls>"},
-	{"<|dsml invoke", "<invoke"},
-	{"</|dsml invoke>", "</invoke>"},
-	{"<|dsml parameter", "<parameter"},
-	{"</|dsml parameter>", "</parameter>"},
-	{"<dsml tool_calls", "<tool_calls"},
-	{"</dsml tool_calls>", "</tool_calls>"},
-	{"<dsml invoke", "<invoke"},
-	{"</dsml invoke>", "</invoke>"},
-	{"<dsml parameter", "<parameter"},
-	{"</dsml parameter>", "</parameter>"},
-	{"<dsml|tool_calls", "<tool_calls"},
-	{"</dsml|tool_calls>", "</tool_calls>"},
-	{"<dsml|invoke", "<invoke"},
-	{"</dsml|invoke>", "</invoke>"},
-	{"<dsml|parameter", "<parameter"},
-	{"</dsml|parameter>", "</parameter>"},
-	{"<|tool_calls", "<tool_calls"},
-	{"</|tool_calls>", "</tool_calls>"},
-	{"<|invoke", "<invoke"},
-	{"</|invoke>", "</invoke>"},
-	{"<|parameter", "<parameter"},
-	{"</|parameter>", "</parameter>"},
-	{"<｜tool_calls", "<tool_calls"},
-	{"</｜tool_calls>", "</tool_calls>"},
-	{"<｜invoke", "<invoke"},
-	{"</｜invoke>", "</invoke>"},
-	{"<｜parameter", "<parameter"},
-	{"</｜parameter>", "</parameter>"},
-}
-
-var canonicalToolMarkupPrefixes = []string{
-	"<tool_calls",
-	"</tool_calls>",
-	"<invoke",
-	"</invoke>",
-	"<parameter",
-	"</parameter>",
-}
-
-func toolMarkupStylesOutsideIgnored(text string) (hasDSML, hasCanonical bool) {
-	lower := strings.ToLower(text)
-	for i := 0; i < len(text); {
-		next, advanced, blocked := skipXMLIgnoredSection(lower, i)
-		if blocked {
-			return hasDSML, hasCanonical
-		}
-		if advanced {
-			i = next
-			continue
-		}
-		if hasPrefixAt(lower, i, canonicalToolMarkupPrefixes) {
-			hasCanonical = true
-		}
-		for _, alias := range dsmlToolMarkupAliases {
-			if strings.HasPrefix(lower[i:], alias.from) {
-				hasDSML = true
-				break
-			}
-		}
-		if hasDSML && hasCanonical {
-			return true, true
-		}
-		i++
+func rewriteDSMLToolMarkupOutsideIgnored(text string) string {
+	if text == "" {
+		return ""
 	}
-	return hasDSML, hasCanonical
-}
-
-func replaceDSMLToolMarkupOutsideIgnored(text string) string {
 	lower := strings.ToLower(text)
 	var b strings.Builder
 	b.Grow(len(text))
@@ -110,29 +31,24 @@ func replaceDSMLToolMarkupOutsideIgnored(text string) string {
 			i = next
 			continue
 		}
-		replaced := false
-		for _, alias := range dsmlToolMarkupAliases {
-			if strings.HasPrefix(lower[i:], alias.from) {
-				b.WriteString(alias.to)
-				i += len(alias.from)
-				replaced = true
-				break
-			}
-		}
-		if replaced {
+		tag, ok := scanToolMarkupTagAt(text, i)
+		if !ok {
+			b.WriteByte(text[i])
+			i++
 			continue
 		}
-		b.WriteByte(text[i])
-		i++
+		if tag.DSMLLike {
+			b.WriteByte('<')
+			if tag.Closing {
+				b.WriteByte('/')
+			}
+			b.WriteString(tag.Name)
+			b.WriteString(text[tag.NameEnd : tag.End+1])
+			i = tag.End + 1
+			continue
+		}
+		b.WriteString(text[tag.Start : tag.End+1])
+		i = tag.End + 1
 	}
 	return b.String()
 }
-
-func hasPrefixAt(text string, idx int, prefixes []string) bool {
-	for _, prefix := range prefixes {
-		if strings.HasPrefix(text[idx:], prefix) {
-			return true
-		}
-	}
-	return false
-}

internal/toolcall/toolcalls_markup.go

@@ -111,5 +111,72 @@ func extractStandaloneCDATA(inner string) (string, bool) {
 	if cdataMatches := cdataPattern.FindStringSubmatch(trimmed); len(cdataMatches) >= 2 {
 		return cdataMatches[1], true
 	}
+	if strings.HasPrefix(strings.ToLower(trimmed), "<![cdata[") {
+		return trimmed[len("<![CDATA["):], true
+	}
 	return "", false
 }
+
+func parseJSONLiteralValue(raw string) (any, bool) {
+	trimmed := strings.TrimSpace(raw)
+	if trimmed == "" {
+		return nil, false
+	}
+
+	switch trimmed[0] {
+	case '{', '[', '"', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 't', 'f', 'n':
+	default:
+		return nil, false
+	}
+
+	var parsed any
+	if err := json.Unmarshal([]byte(trimmed), &parsed); err != nil {
+		return nil, false
+	}
+	return parsed, true
+}
+
+// SanitizeLooseCDATA repairs malformed trailing CDATA openings just enough for
+// final parsing and flush-time recovery. Properly closed CDATA blocks are left
+// untouched; an unclosed opener is stripped so the remaining text can still be
+// parsed as part of the surrounding tool markup.
+func SanitizeLooseCDATA(text string) string {
+	if text == "" {
+		return ""
+	}
+
+	lower := strings.ToLower(text)
+	const openMarker = "<![cdata["
+	const closeMarker = "]]>"
+
+	var b strings.Builder
+	b.Grow(len(text))
+	changed := false
+	pos := 0
+	for pos < len(text) {
+		startRel := strings.Index(lower[pos:], openMarker)
+		if startRel < 0 {
+			b.WriteString(text[pos:])
+			break
+		}
+		start := pos + startRel
+		contentStart := start + len(openMarker)
+		b.WriteString(text[pos:start])
+
+		if endRel := strings.Index(lower[contentStart:], closeMarker); endRel >= 0 {
+			end := contentStart + endRel + len(closeMarker)
+			b.WriteString(text[start:end])
+			pos = end
+			continue
+		}
+
+		changed = true
+		b.WriteString(text[contentStart:])
+		pos = len(text)
+	}
+
+	if !changed {
+		return text
+	}
+	return b.String()
+}

internal/toolcall/toolcalls_parse.go

@@ -65,6 +65,12 @@ func parseToolCallsDetailedXMLOnly(text string) ToolCallParseResult {
 		return result
 	}
 	parsed := parseXMLToolCalls(normalized)
+	if len(parsed) == 0 && strings.Contains(strings.ToLower(normalized), "<![cdata[") {
+		recovered := SanitizeLooseCDATA(normalized)
+		if recovered != normalized {
+			parsed = parseXMLToolCalls(recovered)
+		}
+	}
 	if len(parsed) == 0 {
 		return result
 	}
@@ -92,14 +98,8 @@ func filterToolCallsDetailed(parsed []ParsedToolCall) ([]ParsedToolCall, []strin
 }
 
 func looksLikeToolCallSyntax(text string) bool {
-	lower := strings.ToLower(text)
-	return strings.Contains(lower, "<|dsml|tool_calls") ||
-		strings.Contains(lower, "<|dsml tool_calls") ||
-		strings.Contains(lower, "<dsml|tool_calls") ||
-		strings.Contains(lower, "<dsml tool_calls") ||
-		strings.Contains(lower, "<｜tool_calls") ||
-		strings.Contains(lower, "<|tool_calls") ||
-		strings.Contains(lower, "<tool_calls")
+	hasDSML, hasCanonical := ContainsToolCallWrapperSyntaxOutsideIgnored(text)
+	return hasDSML || hasCanonical
 }
 
 func stripFencedCodeBlocks(text string) string {

internal/toolcall/toolcalls_parse_markup.go

@@ -295,15 +295,24 @@ func parseInvokeParameterValue(raw string) any {
 		return ""
 	}
 	if value, ok := extractStandaloneCDATA(trimmed); ok {
+		if parsed, ok := parseJSONLiteralValue(value); ok {
+			return parsed
+		}
 		return value
 	}
-	if parsed := parseStructuredToolCallInput(trimmed); len(parsed) > 0 {
-		if len(parsed) == 1 {
-			if rawValue, ok := parsed["_raw"].(string); ok {
-				return rawValue
+	decoded := html.UnescapeString(extractRawTagValue(trimmed))
+	if strings.Contains(decoded, "<") && strings.Contains(decoded, ">") {
+		if parsed := parseStructuredToolCallInput(decoded); len(parsed) > 0 {
+			if len(parsed) == 1 {
+				if rawValue, ok := parsed["_raw"].(string); ok {
+					return rawValue
+				}
 			}
+			return parsed
 		}
+	}
+	if parsed, ok := parseJSONLiteralValue(decoded); ok {
 		return parsed
 	}
-	return html.UnescapeString(extractRawTagValue(trimmed))
+	return decoded
 }

internal/toolcall/toolcalls_scan.go

@@ -0,0 +1,219 @@
+package toolcall
+
+import "strings"
+
+var toolMarkupNames = []string{"tool_calls", "invoke", "parameter"}
+
+type ToolMarkupTag struct {
+	Start       int
+	End         int
+	NameStart   int
+	NameEnd     int
+	Name        string
+	Closing     bool
+	SelfClosing bool
+	DSMLLike    bool
+	Canonical   bool
+}
+
+func ContainsToolMarkupSyntaxOutsideIgnored(text string) (hasDSML, hasCanonical bool) {
+	lower := strings.ToLower(text)
+	for i := 0; i < len(text); {
+		next, advanced, blocked := skipXMLIgnoredSection(lower, i)
+		if blocked {
+			return hasDSML, hasCanonical
+		}
+		if advanced {
+			i = next
+			continue
+		}
+		if tag, ok := scanToolMarkupTagAt(text, i); ok {
+			if tag.DSMLLike {
+				hasDSML = true
+			} else {
+				hasCanonical = true
+			}
+			if hasDSML && hasCanonical {
+				return true, true
+			}
+			i = tag.End + 1
+			continue
+		}
+		i++
+	}
+	return hasDSML, hasCanonical
+}
+
+func ContainsToolCallWrapperSyntaxOutsideIgnored(text string) (hasDSML, hasCanonical bool) {
+	lower := strings.ToLower(text)
+	for i := 0; i < len(text); {
+		next, advanced, blocked := skipXMLIgnoredSection(lower, i)
+		if blocked {
+			return hasDSML, hasCanonical
+		}
+		if advanced {
+			i = next
+			continue
+		}
+		if tag, ok := scanToolMarkupTagAt(text, i); ok {
+			if tag.Name != "tool_calls" {
+				i = tag.End + 1
+				continue
+			}
+			if tag.DSMLLike {
+				hasDSML = true
+			} else {
+				hasCanonical = true
+			}
+			if hasDSML && hasCanonical {
+				return true, true
+			}
+			i = tag.End + 1
+			continue
+		}
+		i++
+	}
+	return hasDSML, hasCanonical
+}
+
+func FindToolMarkupTagOutsideIgnored(text string, start int) (ToolMarkupTag, bool) {
+	lower := strings.ToLower(text)
+	for i := maxInt(start, 0); i < len(text); {
+		next, advanced, blocked := skipXMLIgnoredSection(lower, i)
+		if blocked {
+			return ToolMarkupTag{}, false
+		}
+		if advanced {
+			i = next
+			continue
+		}
+		if tag, ok := scanToolMarkupTagAt(text, i); ok {
+			return tag, true
+		}
+		i++
+	}
+	return ToolMarkupTag{}, false
+}
+
+func FindMatchingToolMarkupClose(text string, open ToolMarkupTag) (ToolMarkupTag, bool) {
+	if text == "" || open.Name == "" || open.Closing {
+		return ToolMarkupTag{}, false
+	}
+	depth := 1
+	for pos := open.End + 1; pos < len(text); {
+		tag, ok := FindToolMarkupTagOutsideIgnored(text, pos)
+		if !ok {
+			return ToolMarkupTag{}, false
+		}
+		if tag.Name != open.Name {
+			pos = tag.End + 1
+			continue
+		}
+		if tag.Closing {
+			depth--
+			if depth == 0 {
+				return tag, true
+			}
+		} else if !tag.SelfClosing {
+			depth++
+		}
+		pos = tag.End + 1
+	}
+	return ToolMarkupTag{}, false
+}
+
+func scanToolMarkupTagAt(text string, start int) (ToolMarkupTag, bool) {
+	if start < 0 || start >= len(text) || text[start] != '<' {
+		return ToolMarkupTag{}, false
+	}
+	lower := strings.ToLower(text)
+	i := start + 1
+	closing := false
+	if i < len(text) && text[i] == '/' {
+		closing = true
+		i++
+	}
+	dsmlLike := false
+	if next, ok := consumeToolMarkupPipe(text, i); ok {
+		dsmlLike = true
+		i = next
+	}
+	if strings.HasPrefix(lower[i:], "dsml") {
+		dsmlLike = true
+		i += len("dsml")
+		for next, ok := consumeToolMarkupSeparator(text, i); ok; next, ok = consumeToolMarkupSeparator(text, i) {
+			i = next
+		}
+	}
+	name, nameLen := matchToolMarkupName(lower, i)
+	if nameLen == 0 {
+		return ToolMarkupTag{}, false
+	}
+	nameEnd := i + nameLen
+	if !hasToolMarkupBoundary(text, nameEnd) {
+		return ToolMarkupTag{}, false
+	}
+	end := findXMLTagEnd(text, nameEnd)
+	if end < 0 {
+		return ToolMarkupTag{}, false
+	}
+	trimmed := strings.TrimSpace(text[start : end+1])
+	return ToolMarkupTag{
+		Start:       start,
+		End:         end,
+		NameStart:   i,
+		NameEnd:     nameEnd,
+		Name:        name,
+		Closing:     closing,
+		SelfClosing: strings.HasSuffix(trimmed, "/>"),
+		DSMLLike:    dsmlLike,
+		Canonical:   !dsmlLike,
+	}, true
+}
+
+func matchToolMarkupName(lower string, start int) (string, int) {
+	for _, name := range toolMarkupNames {
+		if strings.HasPrefix(lower[start:], name) {
+			return name, len(name)
+		}
+	}
+	return "", 0
+}
+
+func consumeToolMarkupPipe(text string, idx int) (int, bool) {
+	if idx >= len(text) {
+		return idx, false
+	}
+	if text[idx] == '|' {
+		return idx + 1, true
+	}
+	if strings.HasPrefix(text[idx:], "｜") {
+		return idx + len("｜"), true
+	}
+	return idx, false
+}
+
+func consumeToolMarkupSeparator(text string, idx int) (int, bool) {
+	if idx >= len(text) {
+		return idx, false
+	}
+	if text[idx] == ' ' || text[idx] == '\t' || text[idx] == '\r' || text[idx] == '\n' {
+		return idx + 1, true
+	}
+	if next, ok := consumeToolMarkupPipe(text, idx); ok {
+		return next, true
+	}
+	return idx, false
+}
+
+func hasToolMarkupBoundary(text string, idx int) bool {
+	if idx >= len(text) {
+		return true
+	}
+	switch text[idx] {
+	case ' ', '\t', '\n', '\r', '>', '/':
+		return true
+	default:
+		return false
+	}
+}

internal/toolcall/toolcalls_test.go

@@ -53,6 +53,18 @@ func TestParseToolCallsSupportsDSMLShellWithCanonicalExampleInCDATA(t *testing.T
 	}
 }
 
+func TestParseToolCallsTreatsUnclosedCDATAAsText(t *testing.T) {
+	text := `<tool_calls><invoke name="Write"><parameter name="content"><![CDATA[hello world</parameter></invoke></tool_calls>`
+	res := ParseToolCallsDetailed(text, []string{"Write"})
+	if len(res.Calls) != 1 {
+		t.Fatalf("expected unclosed CDATA to still parse via outer wrapper, got %#v", res.Calls)
+	}
+	got, _ := res.Calls[0].Input["content"].(string)
+	if got != "hello world" {
+		t.Fatalf("expected recovered CDATA payload, got %q", got)
+	}
+}
+
 func TestParseToolCallsNormalizesMixedDSMLAndCanonicalToolTags(t *testing.T) {
 	// Models commonly mix DSML wrapper tags with canonical inner tags.
 	// These should be normalized and parsed, not rejected.
@@ -130,6 +142,23 @@ func TestParseToolCallsSupportsInvokeParameters(t *testing.T) {
 	}
 }
 
+func TestParseToolCallsSupportsJSONScalarParameters(t *testing.T) {
+	text := `<tool_calls><invoke name="configure"><parameter name="count">123</parameter><parameter name="max_tokens"><![CDATA[256]]></parameter><parameter name="enabled">true</parameter></invoke></tool_calls>`
+	calls := ParseToolCalls(text, []string{"configure"})
+	if len(calls) != 1 {
+		t.Fatalf("expected 1 call, got %#v", calls)
+	}
+	if got, ok := calls[0].Input["count"].(float64); !ok || got != 123 {
+		t.Fatalf("expected numeric count, got %#v", calls[0].Input["count"])
+	}
+	if got, ok := calls[0].Input["max_tokens"].(float64); !ok || got != 256 {
+		t.Fatalf("expected numeric max_tokens, got %#v", calls[0].Input["max_tokens"])
+	}
+	if got, ok := calls[0].Input["enabled"].(bool); !ok || !got {
+		t.Fatalf("expected boolean enabled, got %#v", calls[0].Input["enabled"])
+	}
+}
+
 func TestParseToolCallsPreservesRawMalformedParams(t *testing.T) {
 	text := `<tool_calls><invoke name="execute_command"><parameter name="command">cd /root && git status</parameter></invoke></tool_calls>`
 	calls := ParseToolCalls(text, []string{"execute_command"})
@@ -478,6 +507,49 @@ func TestParseToolCallsDoesNotAcceptDSMLSpaceLookalikeTagName(t *testing.T) {
 	}
 }
 
+func TestParseToolCallsToleratesDSMLCollapsedTagNames(t *testing.T) {
+	todos := `[x] 检查 toolcalls_format.go 格式化逻辑
+[x] 检查 toolcalls_parse.go 解析逻辑
+[x] 检查 toolcalls_xml.go 和 toolcalls_dsml.go
+[x] 检查 toolcalls_markup.go 和 toolcalls_json_repair.go
+[x] 检查 prompt/tool_calls.go 注入逻辑
+[x] 检查 toolstream 流式解析
+[x] 查看测试文件确认预期行为
+[x] 给出调查结论`
+	text := strings.Join([]string{
+		"[]",
+		"<DSMLtool_calls>",
+		"<DSMLinvoke name=\"update_todo_list\">",
+		"<DSMLparameter name=\"todos\"><![CDATA[" + todos + "]]></DSMLparameter>",
+		"</DSMLinvoke>",
+		"</DSMLtool_calls>",
+	}, "\n")
+	calls := ParseToolCalls(text, []string{"update_todo_list"})
+	if len(calls) != 1 {
+		t.Fatalf("expected one call from collapsed DSML tags, got %#v", calls)
+	}
+	if calls[0].Name != "update_todo_list" {
+		t.Fatalf("expected update_todo_list call, got %#v", calls[0])
+	}
+	if got, _ := calls[0].Input["todos"].(string); got != todos {
+		t.Fatalf("expected todos to round-trip, got %q", got)
+	}
+}
+
+func TestParseToolCallsDoesNotAcceptDSMLCollapsedLookalikeTagName(t *testing.T) {
+	text := strings.Join([]string{
+		"<DSMLtool_calls_extra>",
+		"<DSMLinvoke name=\"update_todo_list\">",
+		"<DSMLparameter name=\"todos\">x</DSMLparameter>",
+		"</DSMLinvoke>",
+		"</DSMLtool_calls_extra>",
+	}, "\n")
+	calls := ParseToolCalls(text, []string{"update_todo_list"})
+	if len(calls) != 0 {
+		t.Fatalf("expected no calls from collapsed lookalike tag, got %#v", calls)
+	}
+}
+
 func TestParseToolCallsSkipsProseMentionOfSameWrapperVariant(t *testing.T) {
 	text := strings.Join([]string{
 		"Summary: support canonical <tool_calls> and DSML <|DSML|tool_calls> wrappers.",