工具优化

This commit is contained in:
CJACK
2026-04-26 09:44:59 +08:00
parent 0bfddf7943
commit 0fb1bc6611
9 changed files with 631 additions and 48 deletions

View File

@@ -1,8 +1,5 @@
'use strict';
const TOOLS_WRAPPER_PATTERN = /<tool_calls\b[^>]*>([\s\S]*?)<\/tool_calls>/gi;
const TOOL_CALL_MARKUP_BLOCK_PATTERN = /<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?invoke>/gi;
const PARAMETER_BLOCK_PATTERN = /<(?:[a-z0-9_:-]+:)?parameter\b([^>]*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?parameter>/gi;
const TOOL_CALL_MARKUP_KV_PATTERN = /<(?:[a-z0-9_:-]+:)?([a-z0-9_.-]+)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi;
const CDATA_PATTERN = /^<!\[CDATA\[([\s\S]*?)]]>$/i;
const XML_ATTR_PATTERN = /\b([a-z0-9_:-]+)\s*=\s*("([^"]*)"|'([^']*)')/gi;
@@ -25,9 +22,9 @@ function parseMarkupToolCalls(text) {
return [];
}
const out = [];
for (const wrapper of raw.matchAll(TOOLS_WRAPPER_PATTERN)) {
const body = toStringSafe(wrapper[1]);
for (const block of body.matchAll(TOOL_CALL_MARKUP_BLOCK_PATTERN)) {
for (const wrapper of findXmlElementBlocks(raw, 'tool_calls')) {
const body = toStringSafe(wrapper.body);
for (const block of findXmlElementBlocks(body, 'invoke')) {
const parsed = parseMarkupSingleToolCall(block);
if (parsed) {
out.push(parsed);
@@ -38,12 +35,12 @@ function parseMarkupToolCalls(text) {
}
function parseMarkupSingleToolCall(block) {
const attrs = parseTagAttributes(block[1]);
const attrs = parseTagAttributes(block.attrs);
const name = toStringSafe(attrs.name).trim();
if (!name) {
return null;
}
const inner = toStringSafe(block[2]).trim();
const inner = toStringSafe(block.body).trim();
if (inner) {
try {
@@ -63,13 +60,13 @@ function parseMarkupSingleToolCall(block) {
}
}
const input = {};
for (const match of inner.matchAll(PARAMETER_BLOCK_PATTERN)) {
const parameterAttrs = parseTagAttributes(match[1]);
for (const match of findXmlElementBlocks(inner, 'parameter')) {
const parameterAttrs = parseTagAttributes(match.attrs);
const paramName = toStringSafe(parameterAttrs.name).trim();
if (!paramName) {
continue;
}
appendMarkupValue(input, paramName, parseMarkupValue(match[2]));
appendMarkupValue(input, paramName, parseMarkupValue(match.body));
}
if (Object.keys(input).length === 0 && inner.trim() !== '') {
return null;
@@ -77,6 +74,154 @@ function parseMarkupSingleToolCall(block) {
return { name, input };
}
function findXmlElementBlocks(text, tag) {
const source = toStringSafe(text);
const name = toStringSafe(tag).toLowerCase();
if (!source || !name) {
return [];
}
const out = [];
let pos = 0;
while (pos < source.length) {
const start = findXmlStartTagOutsideCDATA(source, name, pos);
if (!start) {
break;
}
const end = findMatchingXmlEndTagOutsideCDATA(source, name, start.bodyStart);
if (!end) {
break;
}
out.push({
attrs: start.attrs,
body: source.slice(start.bodyStart, end.closeStart),
start: start.start,
end: end.closeEnd,
});
pos = end.closeEnd;
}
return out;
}
function findXmlStartTagOutsideCDATA(text, tag, from) {
const lower = text.toLowerCase();
const target = `<${tag}`;
for (let i = Math.max(0, from || 0); i < text.length;) {
const skipped = skipXmlIgnoredSection(lower, i);
if (skipped.blocked) {
return null;
}
if (skipped.advanced) {
i = skipped.next;
continue;
}
if (lower.startsWith(target, i) && hasXmlTagBoundary(text, i + target.length)) {
const tagEnd = findXmlTagEnd(text, i + target.length);
if (tagEnd < 0) {
return null;
}
return {
start: i,
bodyStart: tagEnd + 1,
attrs: text.slice(i + target.length, tagEnd),
};
}
i += 1;
}
return null;
}
function findMatchingXmlEndTagOutsideCDATA(text, tag, from) {
const lower = text.toLowerCase();
const openTarget = `<${tag}`;
const closeTarget = `</${tag}`;
let depth = 1;
for (let i = Math.max(0, from || 0); i < text.length;) {
const skipped = skipXmlIgnoredSection(lower, i);
if (skipped.blocked) {
return null;
}
if (skipped.advanced) {
i = skipped.next;
continue;
}
if (lower.startsWith(closeTarget, i) && hasXmlTagBoundary(text, i + closeTarget.length)) {
const tagEnd = findXmlTagEnd(text, i + closeTarget.length);
if (tagEnd < 0) {
return null;
}
depth -= 1;
if (depth === 0) {
return { closeStart: i, closeEnd: tagEnd + 1 };
}
i = tagEnd + 1;
continue;
}
if (lower.startsWith(openTarget, i) && hasXmlTagBoundary(text, i + openTarget.length)) {
const tagEnd = findXmlTagEnd(text, i + openTarget.length);
if (tagEnd < 0) {
return null;
}
if (!isSelfClosingXmlTag(text.slice(i, tagEnd))) {
depth += 1;
}
i = tagEnd + 1;
continue;
}
i += 1;
}
return null;
}
function skipXmlIgnoredSection(lower, i) {
if (lower.startsWith('<![cdata[', i)) {
const end = lower.indexOf(']]>', i + '<![cdata['.length);
if (end < 0) {
return { advanced: false, blocked: true, next: i };
}
return { advanced: true, blocked: false, next: end + ']]>'.length };
}
if (lower.startsWith('<!--', i)) {
const end = lower.indexOf('-->', i + '<!--'.length);
if (end < 0) {
return { advanced: false, blocked: true, next: i };
}
return { advanced: true, blocked: false, next: end + '-->'.length };
}
return { advanced: false, blocked: false, next: i };
}
function findXmlTagEnd(text, from) {
let quote = '';
for (let i = Math.max(0, from || 0); i < text.length; i += 1) {
const ch = text[i];
if (quote) {
if (ch === quote) {
quote = '';
}
continue;
}
if (ch === '"' || ch === "'") {
quote = ch;
continue;
}
if (ch === '>') {
return i;
}
}
return -1;
}
function hasXmlTagBoundary(text, idx) {
if (idx >= text.length) {
return true;
}
return [' ', '\t', '\n', '\r', '>', '/'].includes(text[idx]);
}
function isSelfClosingXmlTag(startTag) {
return toStringSafe(startTag).trim().endsWith('/');
}
function parseMarkupInput(raw) {
const s = toStringSafe(raw).trim();
if (!s) {
@@ -120,6 +265,10 @@ function parseMarkupKVObject(text) {
}
function parseMarkupValue(raw) {
const cdata = extractStandaloneCDATA(raw);
if (cdata.ok) {
return cdata.value;
}
const s = toStringSafe(extractRawTagValue(raw)).trim();
if (!s) {
return '';
@@ -152,9 +301,9 @@ function extractRawTagValue(inner) {
}
// 1. Check for CDATA
const cdataMatch = s.match(CDATA_PATTERN);
if (cdataMatch && cdataMatch[1] !== undefined) {
return cdataMatch[1];
const cdata = extractStandaloneCDATA(s);
if (cdata.ok) {
return cdata.value;
}
// 2. Fallback to unescaping standard HTML entities
@@ -172,6 +321,15 @@ function unescapeHtml(safe) {
.replace(/&#x27;/g, "'");
}
function extractStandaloneCDATA(inner) {
const s = toStringSafe(inner).trim();
const cdataMatch = s.match(CDATA_PATTERN);
if (cdataMatch && cdataMatch[1] !== undefined) {
return { ok: true, value: cdataMatch[1] };
}
return { ok: false, value: '' };
}
function parseTagAttributes(raw) {
const source = toStringSafe(raw);
const out = {};

View File

@@ -16,9 +16,10 @@ function consumeXMLToolCapture(captured, toolNames, trimWrappingJSONFence) {
if (openIdx < 0) {
continue;
}
// Find the LAST occurrence of the specific closing tag.
const closeIdx = lower.lastIndexOf(pair.close);
if (closeIdx < openIdx) {
// Ignore closing tags that appear inside CDATA payloads, such as
// write-file content containing tool-call documentation examples.
const closeIdx = findXMLCloseOutsideCDATA(captured, pair.close, openIdx + pair.open.length);
if (closeIdx < 0) {
// Opening tag present but specific closing tag hasn't arrived.
// Return not-ready so buffering continues until the wrapper closes.
return { ready: false, prefix: '', calls: [], suffix: '' };
@@ -46,8 +47,9 @@ function consumeXMLToolCapture(captured, toolNames, trimWrappingJSONFence) {
function hasOpenXMLToolTag(captured) {
const lower = captured.toLowerCase();
for (const pair of XML_TOOL_TAG_PAIRS) {
if (lower.includes(pair.open)) {
if (!lower.includes(pair.close)) {
const openIdx = lower.indexOf(pair.open);
if (openIdx >= 0) {
if (findXMLCloseOutsideCDATA(captured, pair.close, openIdx + pair.open.length) < 0) {
return true;
}
}
@@ -74,6 +76,38 @@ function findPartialXMLToolTagStart(s) {
return -1;
}
function findXMLCloseOutsideCDATA(s, closeTag, start) {
const text = typeof s === 'string' ? s : '';
const target = String(closeTag || '').toLowerCase();
if (!text || !target) {
return -1;
}
const lower = text.toLowerCase();
for (let i = Math.max(0, start || 0); i < text.length;) {
if (lower.startsWith('<![cdata[', i)) {
const end = lower.indexOf(']]>', i + '<![cdata['.length);
if (end < 0) {
return -1;
}
i = end + ']]>'.length;
continue;
}
if (lower.startsWith('<!--', i)) {
const end = lower.indexOf('-->', i + '<!--'.length);
if (end < 0) {
return -1;
}
i = end + '-->'.length;
continue;
}
if (lower.startsWith(target, i)) {
return i;
}
i += 1;
}
return -1;
}
module.exports = {
consumeXMLToolCapture,
hasOpenXMLToolTag,

View File

@@ -43,6 +43,9 @@ func parseMarkupKVObject(text string) map[string]any {
}
func parseMarkupValue(inner string) any {
if value, ok := extractStandaloneCDATA(inner); ok {
return value
}
value := strings.TrimSpace(extractRawTagValue(inner))
if value == "" {
return ""
@@ -89,8 +92,8 @@ func extractRawTagValue(inner string) string {
}
// 1. Check for CDATA - if present, it's the ultimate "safe" container.
if cdataMatches := cdataPattern.FindStringSubmatch(trimmed); len(cdataMatches) >= 2 {
return cdataMatches[1] // Return raw content between CDATA brackets
if value, ok := extractStandaloneCDATA(trimmed); ok {
return value // Return raw content between CDATA brackets
}
// 2. If no CDATA, we still want to be robust.
@@ -102,3 +105,11 @@ func extractRawTagValue(inner string) string {
// but for KV objects we usually want the value.
return html.UnescapeString(inner)
}
func extractStandaloneCDATA(inner string) (string, bool) {
trimmed := strings.TrimSpace(inner)
if cdataMatches := cdataPattern.FindStringSubmatch(trimmed); len(cdataMatches) >= 2 {
return cdataMatches[1], true
}
return "", false
}

View File

@@ -87,7 +87,13 @@ func stripFencedCodeBlocks(text string) string {
lines := strings.SplitAfter(text, "\n")
inFence := false
fenceMarker := ""
inCDATA := false
for _, line := range lines {
if inCDATA || cdataStartsBeforeFence(line) {
b.WriteString(line)
inCDATA = updateCDATAState(inCDATA, line)
continue
}
trimmed := strings.TrimLeft(line, " \t")
if !inFence {
if marker, ok := parseFenceOpen(trimmed); ok {
@@ -111,6 +117,54 @@ func stripFencedCodeBlocks(text string) string {
return b.String()
}
func cdataStartsBeforeFence(line string) bool {
cdataIdx := strings.Index(strings.ToLower(line), "<![cdata[")
if cdataIdx < 0 {
return false
}
fenceIdx := firstFenceMarkerIndex(line)
return fenceIdx < 0 || cdataIdx < fenceIdx
}
func firstFenceMarkerIndex(line string) int {
idxBacktick := strings.Index(line, "```")
idxTilde := strings.Index(line, "~~~")
switch {
case idxBacktick < 0:
return idxTilde
case idxTilde < 0:
return idxBacktick
case idxBacktick < idxTilde:
return idxBacktick
default:
return idxTilde
}
}
func updateCDATAState(inCDATA bool, line string) bool {
lower := strings.ToLower(line)
pos := 0
state := inCDATA
for pos < len(lower) {
if state {
end := strings.Index(lower[pos:], "]]>")
if end < 0 {
return true
}
pos += end + len("]]>")
state = false
continue
}
start := strings.Index(lower[pos:], "<![cdata[")
if start < 0 {
return false
}
pos += start + len("<![cdata[")
state = true
}
return state
}
func parseFenceOpen(line string) (string, bool) {
if len(line) < 3 {
return "", false

View File

@@ -7,19 +7,16 @@ import (
"strings"
)
var xmlToolCallsWrapperPattern = regexp.MustCompile(`(?is)<tool_calls\b[^>]*>\s*(.*?)\s*</tool_calls>`)
var xmlInvokePattern = regexp.MustCompile(`(?is)<invoke\b([^>]*)>\s*(.*?)\s*</invoke>`)
var xmlParameterPattern = regexp.MustCompile(`(?is)<parameter\b([^>]*)>\s*(.*?)\s*</parameter>`)
var xmlAttrPattern = regexp.MustCompile(`(?is)\b([a-z0-9_:-]+)\s*=\s*("([^"]*)"|'([^']*)')`)
var xmlToolCallsClosePattern = regexp.MustCompile(`(?is)</tool_calls>`)
var xmlInvokeStartPattern = regexp.MustCompile(`(?is)<invoke\b[^>]*\bname\s*=\s*("([^"]*)"|'([^']*)')`)
func parseXMLToolCalls(text string) []ParsedToolCall {
wrappers := xmlToolCallsWrapperPattern.FindAllStringSubmatch(text, -1)
wrappers := findXMLElementBlocks(text, "tool_calls")
if len(wrappers) == 0 {
repaired := repairMissingXMLToolCallsOpeningWrapper(text)
if repaired != text {
wrappers = xmlToolCallsWrapperPattern.FindAllStringSubmatch(repaired, -1)
wrappers = findXMLElementBlocks(repaired, "tool_calls")
}
}
if len(wrappers) == 0 {
@@ -27,10 +24,7 @@ func parseXMLToolCalls(text string) []ParsedToolCall {
}
out := make([]ParsedToolCall, 0, len(wrappers))
for _, wrapper := range wrappers {
if len(wrapper) < 2 {
continue
}
for _, block := range xmlInvokePattern.FindAllStringSubmatch(wrapper[1], -1) {
for _, block := range findXMLElementBlocks(wrapper.Body, "invoke") {
call, ok := parseSingleXMLToolCall(block)
if !ok {
continue
@@ -66,17 +60,14 @@ func repairMissingXMLToolCallsOpeningWrapper(text string) string {
return text[:invokeLoc[0]] + "<tool_calls>" + text[invokeLoc[0]:closeLoc[0]] + "</tool_calls>" + text[closeLoc[1]:]
}
func parseSingleXMLToolCall(block []string) (ParsedToolCall, bool) {
if len(block) < 3 {
return ParsedToolCall{}, false
}
attrs := parseXMLTagAttributes(block[1])
func parseSingleXMLToolCall(block xmlElementBlock) (ParsedToolCall, bool) {
attrs := parseXMLTagAttributes(block.Attrs)
name := strings.TrimSpace(html.UnescapeString(attrs["name"]))
if name == "" {
return ParsedToolCall{}, false
}
inner := strings.TrimSpace(block[2])
inner := strings.TrimSpace(block.Body)
if strings.HasPrefix(inner, "{") {
var payload map[string]any
if err := json.Unmarshal([]byte(inner), &payload); err == nil {
@@ -94,16 +85,13 @@ func parseSingleXMLToolCall(block []string) (ParsedToolCall, bool) {
}
input := map[string]any{}
for _, paramMatch := range xmlParameterPattern.FindAllStringSubmatch(inner, -1) {
if len(paramMatch) < 3 {
continue
}
paramAttrs := parseXMLTagAttributes(paramMatch[1])
for _, paramMatch := range findXMLElementBlocks(inner, "parameter") {
paramAttrs := parseXMLTagAttributes(paramMatch.Attrs)
paramName := strings.TrimSpace(html.UnescapeString(paramAttrs["name"]))
if paramName == "" {
continue
}
value := parseInvokeParameterValue(paramMatch[2])
value := parseInvokeParameterValue(paramMatch.Body)
appendMarkupValue(input, paramName, value)
}
@@ -116,6 +104,168 @@ func parseSingleXMLToolCall(block []string) (ParsedToolCall, bool) {
return ParsedToolCall{Name: name, Input: input}, true
}
type xmlElementBlock struct {
Attrs string
Body string
Start int
End int
}
func findXMLElementBlocks(text, tag string) []xmlElementBlock {
if text == "" || tag == "" {
return nil
}
var out []xmlElementBlock
pos := 0
for pos < len(text) {
start, bodyStart, attrs, ok := findXMLStartTagOutsideCDATA(text, tag, pos)
if !ok {
break
}
closeStart, closeEnd, ok := findMatchingXMLEndTagOutsideCDATA(text, tag, bodyStart)
if !ok {
break
}
out = append(out, xmlElementBlock{
Attrs: attrs,
Body: text[bodyStart:closeStart],
Start: start,
End: closeEnd,
})
pos = closeEnd
}
return out
}
func findXMLStartTagOutsideCDATA(text, tag string, from int) (start, bodyStart int, attrs string, ok bool) {
lower := strings.ToLower(text)
target := "<" + strings.ToLower(tag)
for i := maxInt(from, 0); i < len(text); {
next, advanced, blocked := skipXMLIgnoredSection(lower, i)
if blocked {
return -1, -1, "", false
}
if advanced {
i = next
continue
}
if strings.HasPrefix(lower[i:], target) && hasXMLTagBoundary(text, i+len(target)) {
end := findXMLTagEnd(text, i+len(target))
if end < 0 {
return -1, -1, "", false
}
return i, end + 1, text[i+len(target) : end], true
}
i++
}
return -1, -1, "", false
}
func findMatchingXMLEndTagOutsideCDATA(text, tag string, from int) (closeStart, closeEnd int, ok bool) {
lower := strings.ToLower(text)
openTarget := "<" + strings.ToLower(tag)
closeTarget := "</" + strings.ToLower(tag)
depth := 1
for i := maxInt(from, 0); i < len(text); {
next, advanced, blocked := skipXMLIgnoredSection(lower, i)
if blocked {
return -1, -1, false
}
if advanced {
i = next
continue
}
if strings.HasPrefix(lower[i:], closeTarget) && hasXMLTagBoundary(text, i+len(closeTarget)) {
end := findXMLTagEnd(text, i+len(closeTarget))
if end < 0 {
return -1, -1, false
}
depth--
if depth == 0 {
return i, end + 1, true
}
i = end + 1
continue
}
if strings.HasPrefix(lower[i:], openTarget) && hasXMLTagBoundary(text, i+len(openTarget)) {
end := findXMLTagEnd(text, i+len(openTarget))
if end < 0 {
return -1, -1, false
}
if !isSelfClosingXMLTag(text[:end]) {
depth++
}
i = end + 1
continue
}
i++
}
return -1, -1, false
}
func skipXMLIgnoredSection(lower string, i int) (next int, advanced bool, blocked bool) {
switch {
case strings.HasPrefix(lower[i:], "<![cdata["):
end := strings.Index(lower[i+len("<![cdata["):], "]]>")
if end < 0 {
return 0, false, true
}
return i + len("<![cdata[") + end + len("]]>"), true, false
case strings.HasPrefix(lower[i:], "<!--"):
end := strings.Index(lower[i+len("<!--"):], "-->")
if end < 0 {
return 0, false, true
}
return i + len("<!--") + end + len("-->"), true, false
default:
return i, false, false
}
}
func findXMLTagEnd(text string, from int) int {
quote := byte(0)
for i := maxInt(from, 0); i < len(text); i++ {
ch := text[i]
if quote != 0 {
if ch == quote {
quote = 0
}
continue
}
if ch == '"' || ch == '\'' {
quote = ch
continue
}
if ch == '>' {
return i
}
}
return -1
}
func hasXMLTagBoundary(text string, idx int) bool {
if idx >= len(text) {
return true
}
switch text[idx] {
case ' ', '\t', '\n', '\r', '>', '/':
return true
default:
return false
}
}
func isSelfClosingXMLTag(startTag string) bool {
return strings.HasSuffix(strings.TrimSpace(startTag), "/")
}
func maxInt(a, b int) int {
if a > b {
return a
}
return b
}
func parseXMLTagAttributes(raw string) map[string]string {
if strings.TrimSpace(raw) == "" {
return map[string]string{}
@@ -143,6 +293,9 @@ func parseInvokeParameterValue(raw string) any {
if trimmed == "" {
return ""
}
if value, ok := extractStandaloneCDATA(trimmed); ok {
return value
}
if parsed := parseStructuredToolCallInput(trimmed); len(parsed) > 0 {
if len(parsed) == 1 {
if rawValue, ok := parsed["_raw"].(string); ok {

View File

@@ -54,6 +54,32 @@ echo "hello"
}
}
func TestParseToolCallsKeepsToolSyntaxInsideCDATAAsParameterText(t *testing.T) {
payload := strings.Join([]string{
"# Release notes",
"",
"```xml",
"<tool_calls>",
" <invoke name=\"demo\">",
" <parameter name=\"value\">x</parameter>",
" </invoke>",
"</tool_calls>",
"```",
}, "\n")
text := `<tool_calls><invoke name="Write"><parameter name="content"><![CDATA[` + payload + `]]></parameter><parameter name="file_path">DS2API-4.0-Release-Notes.md</parameter></invoke></tool_calls>`
calls := ParseToolCalls(text, []string{"Write"})
if len(calls) != 1 {
t.Fatalf("expected 1 call, got %#v", calls)
}
content, _ := calls[0].Input["content"].(string)
if content != payload {
t.Fatalf("expected CDATA payload with nested tool syntax to survive intact, got %q", content)
}
if calls[0].Input["file_path"] != "DS2API-4.0-Release-Notes.md" {
t.Fatalf("expected file_path parameter, got %#v", calls[0].Input)
}
}
func TestParseToolCallsSupportsInvokeParameters(t *testing.T) {
text := `<tool_calls><invoke name="get_weather"><parameter name="city">beijing</parameter><parameter name="unit">c</parameter></invoke></tool_calls>`
calls := ParseToolCalls(text, []string{"get_weather"})

View File

@@ -35,9 +35,10 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
if openIdx < 0 {
continue
}
// Find the LAST occurrence of the specific closing tag to get the outermost block.
closeIdx := strings.LastIndex(lower, pair.close)
if closeIdx < openIdx {
// Find the matching closing tag outside CDATA. Long write-file tool
// calls often contain XML examples in CDATA, including </tool_calls>.
closeIdx := findXMLCloseOutsideCDATA(captured, pair.close, openIdx+len(pair.open))
if closeIdx < 0 {
// Opening tag is present but its specific closing tag hasn't arrived.
// Return not-ready so we keep buffering until the canonical wrapper closes.
return "", nil, "", false
@@ -57,7 +58,7 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
}
if !strings.Contains(lower, "<tool_calls") {
invokeIdx := strings.Index(lower, "<invoke")
closeIdx := strings.LastIndex(lower, "</tool_calls>")
closeIdx := findXMLCloseOutsideCDATA(captured, "</tool_calls>", invokeIdx)
if invokeIdx >= 0 && closeIdx > invokeIdx {
closeEnd := closeIdx + len("</tool_calls>")
xmlBlock := "<tool_calls>" + captured[invokeIdx:closeIdx] + "</tool_calls>"
@@ -79,8 +80,9 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
func hasOpenXMLToolTag(captured string) bool {
lower := strings.ToLower(captured)
for _, pair := range xmlToolCallTagPairs {
if strings.Contains(lower, pair.open) {
if !strings.Contains(lower, pair.close) {
openIdx := strings.Index(lower, pair.open)
if openIdx >= 0 {
if findXMLCloseOutsideCDATA(captured, pair.close, openIdx+len(pair.open)) < 0 {
return true
}
}
@@ -88,6 +90,38 @@ func hasOpenXMLToolTag(captured string) bool {
return false
}
func findXMLCloseOutsideCDATA(s, closeTag string, start int) int {
if s == "" || closeTag == "" {
return -1
}
if start < 0 {
start = 0
}
lower := strings.ToLower(s)
target := strings.ToLower(closeTag)
for i := start; i < len(s); {
switch {
case strings.HasPrefix(lower[i:], "<![cdata["):
end := strings.Index(lower[i+len("<![cdata["):], "]]>")
if end < 0 {
return -1
}
i += len("<![cdata[") + end + len("]]>")
case strings.HasPrefix(lower[i:], "<!--"):
end := strings.Index(lower[i+len("<!--"):], "-->")
if end < 0 {
return -1
}
i += len("<!--") + end + len("-->")
case strings.HasPrefix(lower[i:], target):
return i
default:
i++
}
}
return -1
}
// findPartialXMLToolTagStart checks if the string ends with a partial canonical
// XML wrapper tag (e.g., "<too") and returns the position of the '<'.
func findPartialXMLToolTagStart(s string) int {

View File

@@ -84,6 +84,65 @@ func TestProcessToolSieveHandlesLongXMLToolCall(t *testing.T) {
}
}
func TestProcessToolSieveKeepsCDATAEmbeddedToolClosingBuffered(t *testing.T) {
var state State
payload := strings.Join([]string{
"# DS2API 4.0 更新内容",
"",
strings.Repeat("x", 4096),
"```xml",
"<tool_calls>",
" <invoke name=\"demo\">",
" <parameter name=\"value\">x</parameter>",
" </invoke>",
"</tool_calls>",
"```",
"tail",
}, "\n")
innerClose := strings.Index(payload, "</tool_calls>") + len("</tool_calls>")
chunks := []string{
"<tool_calls>\n <invoke name=\"Write\">\n <parameter name=\"content\"><![CDATA[",
payload[:innerClose],
payload[innerClose:],
"]]></parameter>\n <parameter name=\"file_path\">DS2API-4.0-Release-Notes.md</parameter>\n </invoke>\n</tool_calls>",
}
var events []Event
for i, c := range chunks {
next := ProcessChunk(&state, c, []string{"Write"})
if i <= 1 {
for _, evt := range next {
if evt.Content != "" || len(evt.ToolCalls) > 0 {
t.Fatalf("expected no events before outer closing tag, chunk=%d events=%#v", i, next)
}
}
}
events = append(events, next...)
}
events = append(events, Flush(&state, []string{"Write"})...)
var textContent strings.Builder
var gotPayload string
toolCalls := 0
for _, evt := range events {
textContent.WriteString(evt.Content)
if len(evt.ToolCalls) > 0 {
toolCalls += len(evt.ToolCalls)
gotPayload, _ = evt.ToolCalls[0].Input["content"].(string)
}
}
if toolCalls != 1 {
t.Fatalf("expected one parsed tool call, got %d events=%#v", toolCalls, events)
}
if textContent.Len() != 0 {
t.Fatalf("expected no leaked text, got %q", textContent.String())
}
if gotPayload != payload {
t.Fatalf("expected full CDATA payload to survive intact, got len=%d want=%d", len(gotPayload), len(payload))
}
}
func TestProcessToolSieveXMLWithLeadingText(t *testing.T) {
var state State
// Model outputs some prose then an XML tool call.

View File

@@ -118,6 +118,60 @@ test('sieve keeps long XML tool calls buffered until the closing tag arrives', (
assert.equal(finalCalls[0].input.content, longContent);
});
test('sieve keeps CDATA tool examples buffered until the outer closing tag arrives', () => {
const content = [
'# DS2API 4.0 更新内容',
'',
'x'.repeat(4096),
'```xml',
'<tool_calls>',
' <invoke name="demo">',
' <parameter name="value">x</parameter>',
' </invoke>',
'</tool_calls>',
'```',
'tail',
].join('\n');
const innerClose = content.indexOf('</tool_calls>') + '</tool_calls>'.length;
const state = createToolSieveState();
const chunks = [
'<tool_calls>\n <invoke name="Write">\n <parameter name="content"><![CDATA[',
content.slice(0, innerClose),
content.slice(innerClose),
']]></parameter>\n <parameter name="file_path">DS2API-4.0-Release-Notes.md</parameter>\n </invoke>\n</tool_calls>',
];
const events = [];
chunks.forEach((chunk, idx) => {
const next = processToolSieveChunk(state, chunk, ['Write']);
if (idx <= 1) {
assert.deepEqual(next, []);
}
events.push(...next);
});
events.push(...flushToolSieve(state, ['Write']));
const leakedText = collectText(events);
const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []);
assert.equal(leakedText, '');
assert.equal(finalCalls.length, 1);
assert.equal(finalCalls[0].name, 'Write');
assert.equal(finalCalls[0].input.content, content);
});
test('parseToolCalls keeps XML-looking CDATA content intact', () => {
const content = [
'# Release notes',
'```xml',
'<tool_calls><invoke name="demo"><parameter name="value">x</parameter></invoke></tool_calls>',
'```',
].join('\n');
const payload = `<tool_calls><invoke name="Write"><parameter name="content"><![CDATA[${content}]]></parameter><parameter name="file_path">DS2API-4.0-Release-Notes.md</parameter></invoke></tool_calls>`;
const calls = parseToolCalls(payload, ['Write']);
assert.equal(calls.length, 1);
assert.equal(calls[0].input.content, content);
assert.equal(calls[0].input.file_path, 'DS2API-4.0-Release-Notes.md');
});
test('sieve passes JSON tool_calls payload through as text (XML-only)', () => {
const events = runSieve(
['{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}'],