lisone/ds2api
1
0
Fork 0
mirror of https://github.com/CJackHwang/ds2api.git synced 2026-05-08 18:35:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #457 from NgoQuocViet2001/ai/skipxml-lower-hotpath

Browse Source 
fix(toolcall): avoid lowercasing ignored XML tails
This commit is contained in:
CJACK.
2026-05-08 17:05:28 +08:00
committed by GitHub
parent 00fe18b505 7179b995bb
commit 410efbd70b
2 changed files with 54 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
internal/toolcall/toolcalls_parse_markup.go
View File

@@ -210,16 +210,15 @@ func skipXMLIgnoredSection(text string, i int) (next int, advanced bool, blocked
if i < 0 || i >= len(text) {
return i, false, false
}
tail := strings.ToLower(text[i:])
switch {
case strings.HasPrefix(tail, "<![cdata["):
case hasASCIIPrefixFoldAt(text, i, "<![cdata["):
end := findToolCDATAEnd(text, i+len("<![cdata["))
if end < 0 {
return 0, false, true
}
return end + len("]]>"), true, false
case strings.HasPrefix(tail, "<!--"):
end := strings.Index(tail[len("<!--"):], "-->")
case strings.HasPrefix(text[i:], "<!--"):
end := strings.Index(text[i+len("<!--"):], "-->")
if end < 0 {
return 0, false, true
}
@@ -229,6 +228,25 @@ func skipXMLIgnoredSection(text string, i int) (next int, advanced bool, blocked
}
}
func hasASCIIPrefixFoldAt(text string, start int, prefix string) bool {
if start < 0 || len(text)-start < len(prefix) {
return false
}
for j := 0; j < len(prefix); j++ {
if asciiLower(text[start+j]) != asciiLower(prefix[j]) {
return false
}
}
return true
}
func asciiLower(b byte) byte {
if b >= 'A' && b <= 'Z' {
return b + ('a' - 'A')
}
return b
}
func findToolCDATAEnd(text string, from int) int {
if from < 0 || from >= len(text) {
return -1

32
internal/toolcall/toolcalls_test.go
View File

@@ -949,6 +949,38 @@ func TestSkipXMLIgnoredSectionBoundaryConditions(t *testing.T) {
}
}
func TestSkipXMLIgnoredSectionCommentWithUnicodeKeepsByteOffset(t *testing.T) {
text := "<!-- İ -->x<tool_calls>"
next, adv, blk := skipXMLIgnoredSection(text, 0)
if blk || !adv {
t.Fatalf("skipXMLIgnoredSection() = (%d, %v, %v), want advanced unblocked comment", next, adv, blk)
}
if want := len("<!-- İ -->"); next != want {
t.Fatalf("skipXMLIgnoredSection() next = %d, want %d", next, want)
}
}
func TestSkipXMLIgnoredSectionMatchesCDATAWithoutAllocatingTail(t *testing.T) {
text := "<![cDaTa[<tool_calls>]]><tool_calls>"
next, adv, blk := skipXMLIgnoredSection(text, 0)
if blk || !adv {
t.Fatalf("skipXMLIgnoredSection() = (%d, %v, %v), want advanced unblocked CDATA", next, adv, blk)
}
if want := len("<![cDaTa[<tool_calls>]]>"); next != want {
t.Fatalf("skipXMLIgnoredSection() next = %d, want %d", next, want)
}
tag, ok := FindToolMarkupTagOutsideIgnored(text, 0)
if !ok {
t.Fatal("expected tool tag after skipped CDATA")
}
if tag.Start != next {
t.Fatalf("FindToolMarkupTagOutsideIgnored() start = %d, want %d", tag.Start, next)
}
}
func TestFindToolCDATAEndBoundaryConditions(t *testing.T) {
text := "<![CDATA[hello]]>"