Merge pull request #458 from CJackHwang/dev

Avoid lowercasing ignored XML tails in toolcall

internal/toolcall/toolcalls_parse_markup.go

@@ -210,16 +210,15 @@ func skipXMLIgnoredSection(text string, i int) (next int, advanced bool, blocked
 	if i < 0 || i >= len(text) {
 		return i, false, false
 	}
-	tail := strings.ToLower(text[i:])
 	switch {
-	case strings.HasPrefix(tail, "<![cdata["):
+	case hasASCIIPrefixFoldAt(text, i, "<![cdata["):
 		end := findToolCDATAEnd(text, i+len("<![cdata["))
 		if end < 0 {
 			return 0, false, true
 		}
 		return end + len("]]>"), true, false
-	case strings.HasPrefix(tail, "<!--"):
+	case strings.HasPrefix(text[i:], "<!--"):
-		end := strings.Index(tail[len("<!--"):], "-->")
+		end := strings.Index(text[i+len("<!--"):], "-->")
 		if end < 0 {
 			return 0, false, true
 		}
@@ -229,6 +228,25 @@ func skipXMLIgnoredSection(text string, i int) (next int, advanced bool, blocked
 	}
 }
 
+func hasASCIIPrefixFoldAt(text string, start int, prefix string) bool {
+	if start < 0 || len(text)-start < len(prefix) {
+		return false
+	}
+	for j := 0; j < len(prefix); j++ {
+		if asciiLower(text[start+j]) != asciiLower(prefix[j]) {
+			return false
+		}
+	}
+	return true
+}
+
+func asciiLower(b byte) byte {
+	if b >= 'A' && b <= 'Z' {
+		return b + ('a' - 'A')
+	}
+	return b
+}
+
 func findToolCDATAEnd(text string, from int) int {
 	if from < 0 || from >= len(text) {
 		return -1

internal/toolcall/toolcalls_test.go

@@ -949,6 +949,38 @@ func TestSkipXMLIgnoredSectionBoundaryConditions(t *testing.T) {
 	}
 }
 
+func TestSkipXMLIgnoredSectionCommentWithUnicodeKeepsByteOffset(t *testing.T) {
+	text := "<!-- İ -->x<tool_calls>"
+
+	next, adv, blk := skipXMLIgnoredSection(text, 0)
+	if blk || !adv {
+		t.Fatalf("skipXMLIgnoredSection() = (%d, %v, %v), want advanced unblocked comment", next, adv, blk)
+	}
+	if want := len("<!-- İ -->"); next != want {
+		t.Fatalf("skipXMLIgnoredSection() next = %d, want %d", next, want)
+	}
+}
+
+func TestSkipXMLIgnoredSectionMatchesCDATAWithoutAllocatingTail(t *testing.T) {
+	text := "<![cDaTa[<tool_calls>]]><tool_calls>"
+
+	next, adv, blk := skipXMLIgnoredSection(text, 0)
+	if blk || !adv {
+		t.Fatalf("skipXMLIgnoredSection() = (%d, %v, %v), want advanced unblocked CDATA", next, adv, blk)
+	}
+	if want := len("<![cDaTa[<tool_calls>]]>"); next != want {
+		t.Fatalf("skipXMLIgnoredSection() next = %d, want %d", next, want)
+	}
+
+	tag, ok := FindToolMarkupTagOutsideIgnored(text, 0)
+	if !ok {
+		t.Fatal("expected tool tag after skipped CDATA")
+	}
+	if tag.Start != next {
+		t.Fatalf("FindToolMarkupTagOutsideIgnored() start = %d, want %d", tag.Start, next)
+	}
+}
+
 func TestFindToolCDATAEndBoundaryConditions(t *testing.T) {
 	text := "<![CDATA[hello]]>"