Files
ds2api/internal/toolcall/toolcalls_dsml.go
CJACK fd0ec29991 refactor: generalize DSML tag parsing to tolerate model noise; split tiktoken by build tags
Replace hardcoded DSML typo variant lists in Go/Node tool call parsers with
generalized prefix consumption that tolerates repeated leading <, repeated DSML
prefix noise, and trailing pipe terminators. Split tiktoken-dependent token
counting into a build-tagged file for non-cgo platform compatibility. Add /data
directory to Dockerfile for bind-mount permissions.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-01 15:17:11 +08:00

58 lines
1.1 KiB
Go

package toolcall
import "strings"
func normalizeDSMLToolCallMarkup(text string) (string, bool) {
if text == "" {
return "", true
}
hasAliasLikeMarkup, _ := ContainsToolMarkupSyntaxOutsideIgnored(text)
if !hasAliasLikeMarkup {
return text, true
}
return rewriteDSMLToolMarkupOutsideIgnored(text), true
}
func rewriteDSMLToolMarkupOutsideIgnored(text string) string {
if text == "" {
return ""
}
lower := strings.ToLower(text)
var b strings.Builder
b.Grow(len(text))
for i := 0; i < len(text); {
next, advanced, blocked := skipXMLIgnoredSection(lower, i)
if blocked {
b.WriteString(text[i:])
break
}
if advanced {
b.WriteString(text[i:next])
i = next
continue
}
tag, ok := scanToolMarkupTagAt(text, i)
if !ok {
b.WriteByte(text[i])
i++
continue
}
if tag.DSMLLike {
b.WriteByte('<')
if tag.Closing {
b.WriteByte('/')
}
b.WriteString(tag.Name)
b.WriteString(text[tag.NameEnd : tag.End+1])
if text[tag.End] != '>' {
b.WriteByte('>')
}
i = tag.End + 1
continue
}
b.WriteString(text[tag.Start : tag.End+1])
i = tag.End + 1
}
return b.String()
}