feat: improve CDATA and DSML tag parsing robustness with support for fullwidth-bang, ideographic-comma, and extended quote/separator normalization.

internal/toolcall/tool_prompt.go

@@ -21,6 +21,7 @@ RULES:
 1) Use the <｜DSML｜tool_calls> wrapper format.
 2) Put one or more <｜DSML｜invoke> entries under a single <｜DSML｜tool_calls> root.
 3) Put the tool name in the invoke name attribute: <｜DSML｜invoke name="TOOL_NAME">.
+3a) Tag punctuation alphabet: ASCII < > / = " plus the fullwidth vertical bar ｜.
 4) All string values must use <![CDATA[...]]>, even short ones. This includes code, scripts, file contents, prompts, paths, names, and queries.
 5) Every top-level argument must be a <｜DSML｜parameter name="ARG_NAME">...</｜DSML｜parameter> node.
 6) Objects use nested XML elements inside the parameter body. Arrays may repeat <item> children.

internal/toolcall/tool_prompt_test.go

@@ -133,6 +133,19 @@ func TestBuildToolCallInstructions_RejectsEmptyParametersInPrompt(t *testing.T)
 	}
 }
 
+func TestBuildToolCallInstructions_UsesPositiveTagPunctuationAlphabet(t *testing.T) {
+	out := BuildToolCallInstructions([]string{"Bash"})
+	want := `Tag punctuation alphabet: ASCII < > / = " plus the fullwidth vertical bar ｜.`
+	if !strings.Contains(out, want) {
+		t.Fatalf("expected positive tag punctuation alphabet %q, got: %s", want, out)
+	}
+	for _, bad := range []string{"lookalike", "substitute", "！", "〈", "〉", "“", "”", "、"} {
+		if strings.Contains(out, bad) {
+			t.Fatalf("tool prompt should not include negative punctuation examples %q, got: %s", bad, out)
+		}
+	}
+}
+
 func findInvokeBlocks(text, name string) []string {
 	open := `<｜DSML｜invoke name="` + name + `">`
 	remaining := text

internal/toolcall/toolcalls_dsml.go

@@ -86,7 +86,7 @@ func normalizeToolMarkupTagTailForXML(tail string) string {
 		case '"', '\'':
 			quote = ch
 			b.WriteRune(ch)
-		case '|':
+		case '|', '!':
 			j := i + size
 			for j < len(tail) {
 				next, nextSize := utf8.DecodeRuneInString(tail[j:])

internal/toolcall/toolcalls_markup.go

@@ -10,7 +10,7 @@ import (
 var toolCallMarkupKVPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)>`)
 
 // cdataPattern matches a standalone CDATA section.
-var cdataPattern = regexp.MustCompile(`(?is)^(?:<|〈)!\[CDATA\[(.*?)]](?:>|＞|〉)$`)
+var cdataPattern = regexp.MustCompile(`(?is)^(?:<|〈)(?:!|！)\[CDATA\[(.*?)]](?:>|＞|〉)$`)
 
 func parseMarkupKVObject(text string) map[string]any {
 	matches := toolCallMarkupKVPattern.FindAllStringSubmatch(strings.TrimSpace(text), -1)
@@ -108,15 +108,32 @@ func extractRawTagValue(inner string) string {
 
 func extractStandaloneCDATA(inner string) (string, bool) {
 	trimmed := strings.TrimSpace(inner)
-	if cdataMatches := cdataPattern.FindStringSubmatch(trimmed); len(cdataMatches) >= 2 {
-		return cdataMatches[1], true
-	}
-	if strings.HasPrefix(strings.ToLower(trimmed), "<![cdata[") {
-		return trimmed[len("<![CDATA["):], true
+	if bodyStart, ok := matchToolCDATAOpenAt(trimmed, 0); ok {
+		end := findStandaloneCDATAEnd(trimmed, bodyStart)
+		if end < 0 {
+			return trimmed[bodyStart:], true
+		}
+		return trimmed[bodyStart:end], true
 	}
 	return "", false
 }
 
+func findStandaloneCDATAEnd(text string, from int) int {
+	end := -1
+	for searchFrom := from; searchFrom < len(text); {
+		next := indexToolCDATAClose(text, searchFrom)
+		if next < 0 {
+			break
+		}
+		closeEnd := next + toolCDATACloseLenAt(text, next)
+		if strings.TrimSpace(text[closeEnd:]) == "" {
+			end = next
+		}
+		searchFrom = closeEnd
+	}
+	return end
+}
+
 func parseJSONLiteralValue(raw string) (any, bool) {
 	trimmed := strings.TrimSpace(raw)
 	if trimmed == "" {

internal/toolcall/toolcalls_parse_markup.go

@@ -209,13 +209,14 @@ func skipXMLIgnoredSection(text string, i int) (next int, advanced bool, blocked
 	if i < 0 || i >= len(text) {
 		return i, false, false
 	}
-	switch {
-	case hasASCIIPrefixFoldAt(text, i, "<![cdata["):
-		end := findToolCDATAEnd(text, i+len("<![cdata["))
+	if bodyStart, ok := matchToolCDATAOpenAt(text, i); ok {
+		end := findToolCDATAEnd(text, bodyStart)
 		if end < 0 {
 			return 0, false, true
 		}
 		return end + toolCDATACloseLenAt(text, end), true, false
+	}
+	switch {
 	case strings.HasPrefix(text[i:], "<!--"):
 		end := strings.Index(text[i+len("<!--"):], "-->")
 		if end < 0 {
@@ -227,6 +228,38 @@ func skipXMLIgnoredSection(text string, i int) (next int, advanced bool, blocked
 	}
 }
 
+func matchToolCDATAOpenAt(text string, start int) (int, bool) {
+	i, ok := consumeToolMarkupLessThan(text, start)
+	if !ok {
+		return start, false
+	}
+	for skipped := 0; skipped <= 4 && i < len(text); skipped++ {
+		if cdataLen, ok := matchASCIIPrefixFoldAt(text, i, "[cdata["); ok {
+			return i + cdataLen, true
+		}
+		r, size := utf8.DecodeRuneInString(text[i:])
+		if size <= 0 || !isToolCDATAOpenSeparator(r) {
+			break
+		}
+		i += size
+	}
+	return start, false
+}
+
+func isToolCDATAOpenSeparator(r rune) bool {
+	ch := normalizeFullwidthASCII(r)
+	if ch == 0 || ch == '<' || ch == '>' || ch == '/' || ch == '=' || ch == '"' || ch == '\'' || ch == '[' {
+		return false
+	}
+	if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' {
+		return false
+	}
+	if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') {
+		return false
+	}
+	return true
+}
+
 func hasASCIIPrefixFoldAt(text string, start int, prefix string) bool {
 	_, ok := matchASCIIPrefixFoldAt(text, start, prefix)
 	return ok

internal/toolcall/toolcalls_scan.go

@@ -159,6 +159,9 @@ func scanToolMarkupTagAt(text string, start int) (ToolMarkupTag, bool) {
 		if !ok {
 			return ToolMarkupTag{}, false
 		}
+		if !closing && toolMarkupPrefixContainsSlash(text[prefixStart:fallbackStart]) {
+			closing = true
+		}
 		name = fallbackName
 		i = fallbackStart
 		nameLen = fallbackLen
@@ -461,6 +464,9 @@ func consumeToolMarkupPipe(text string, idx int) (int, bool) {
 	if strings.HasPrefix(text[idx:], "␂") {
 		return idx + len("␂"), true
 	}
+	if ch, size := normalizedASCIIAt(text, idx); ch == '!' {
+		return idx + size, true
+	}
 	return idx, false
 }
 
@@ -506,9 +512,22 @@ func normalizeFullwidthASCII(r rune) rune {
 		return '<'
 	case '〉':
 		return '>'
+	case '“', '”':
+		return '"'
+	case '‘', '’':
+		return '\''
 	}
 	if r >= '！' && r <= '～' {
 		return r - 0xFEE0
 	}
 	return r
 }
+
+func toolMarkupPrefixContainsSlash(prefix string) bool {
+	for _, r := range prefix {
+		if normalizeFullwidthASCII(r) == '/' {
+			return true
+		}
+	}
+	return false
+}

internal/toolcall/toolcalls_test.go

@@ -163,6 +163,49 @@ func TestParseToolCallsSupportsCJKAngleDSMDrift(t *testing.T) {
 	}
 }
 
+func TestParseToolCallsSupportsFullwidthBangDSMLDrift(t *testing.T) {
+	text := `<！DSML！tool_calls>
+  <！DSML！invoke name=“Bash”>
+  <！DSML！parameter name=“command”><！[CDATA[lsof -i :4321 -t]]><！/DSML！parameter>
+  <！DSML！parameter name=“description”><！[CDATA[Verify port 4321 is free]]><！/DSML！parameter>
+  <！/DSML！invoke>
+  <！/DSML！tool_calls>`
+
+	calls := ParseToolCalls(text, []string{"Bash"})
+	if len(calls) != 1 {
+		t.Fatalf("expected one fullwidth-bang DSML drift call, got %#v", calls)
+	}
+	if calls[0].Name != "Bash" || calls[0].Input["command"] != "lsof -i :4321 -t" || calls[0].Input["description"] != "Verify port 4321 is free" {
+		t.Fatalf("unexpected fullwidth-bang DSML drift call: %#v", calls[0])
+	}
+}
+
+func TestParseToolCallsSupportsIdeographicCommaDSMLDrift(t *testing.T) {
+	text := `<、DSML、tool_calls>
+  <、DSML、invoke name="Bash">
+    <、DSML、parameter name="command"><、[CDATA[git commit -m "$(cat <<'EOF'
+feat: expand fullwidth bang separator and curly quote tolerance in DSML tool parsing
+
+Co-Authored-By: Claude Opus 4.6 noreply@anthropic.com
+EOF
+)"]]><、/DSML、parameter>
+    <、DSML、parameter name="description"><、[CDATA[Create commit with staged changes]]><、/DSML、parameter>
+  <、/DSML、invoke>
+<、/DSML、tool_calls>`
+
+	calls := ParseToolCalls(text, []string{"Bash"})
+	if len(calls) != 1 {
+		t.Fatalf("expected one ideographic-comma DSML drift call, got %#v", calls)
+	}
+	command, _ := calls[0].Input["command"].(string)
+	if calls[0].Name != "Bash" || !strings.Contains(command, `git commit -m "$(cat <<'EOF'`) || !strings.Contains(command, "Co-Authored-By: Claude Opus 4.6 noreply@anthropic.com") {
+		t.Fatalf("unexpected ideographic-comma DSML drift call: %#v", calls[0])
+	}
+	if calls[0].Input["description"] != "Create commit with staged changes" {
+		t.Fatalf("unexpected ideographic-comma description: %#v", calls[0])
+	}
+}
+
 func TestParseToolCallsIgnoresBareHyphenatedToolCallsLookalike(t *testing.T) {
 	text := `<tool-calls><invoke name="Bash"><parameter name="command">pwd</parameter></invoke></tool-calls>`
 	calls := ParseToolCalls(text, []string{"Bash"})