mirror of
https://github.com/CJackHwang/ds2api.git
synced 2026-05-04 16:35:27 +08:00
工具优化
This commit is contained in:
@@ -1,8 +1,5 @@
|
||||
'use strict';
|
||||
|
||||
const TOOLS_WRAPPER_PATTERN = /<tool_calls\b[^>]*>([\s\S]*?)<\/tool_calls>/gi;
|
||||
const TOOL_CALL_MARKUP_BLOCK_PATTERN = /<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?invoke>/gi;
|
||||
const PARAMETER_BLOCK_PATTERN = /<(?:[a-z0-9_:-]+:)?parameter\b([^>]*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?parameter>/gi;
|
||||
const TOOL_CALL_MARKUP_KV_PATTERN = /<(?:[a-z0-9_:-]+:)?([a-z0-9_.-]+)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi;
|
||||
const CDATA_PATTERN = /^<!\[CDATA\[([\s\S]*?)]]>$/i;
|
||||
const XML_ATTR_PATTERN = /\b([a-z0-9_:-]+)\s*=\s*("([^"]*)"|'([^']*)')/gi;
|
||||
@@ -25,9 +22,9 @@ function parseMarkupToolCalls(text) {
|
||||
return [];
|
||||
}
|
||||
const out = [];
|
||||
for (const wrapper of raw.matchAll(TOOLS_WRAPPER_PATTERN)) {
|
||||
const body = toStringSafe(wrapper[1]);
|
||||
for (const block of body.matchAll(TOOL_CALL_MARKUP_BLOCK_PATTERN)) {
|
||||
for (const wrapper of findXmlElementBlocks(raw, 'tool_calls')) {
|
||||
const body = toStringSafe(wrapper.body);
|
||||
for (const block of findXmlElementBlocks(body, 'invoke')) {
|
||||
const parsed = parseMarkupSingleToolCall(block);
|
||||
if (parsed) {
|
||||
out.push(parsed);
|
||||
@@ -38,12 +35,12 @@ function parseMarkupToolCalls(text) {
|
||||
}
|
||||
|
||||
function parseMarkupSingleToolCall(block) {
|
||||
const attrs = parseTagAttributes(block[1]);
|
||||
const attrs = parseTagAttributes(block.attrs);
|
||||
const name = toStringSafe(attrs.name).trim();
|
||||
if (!name) {
|
||||
return null;
|
||||
}
|
||||
const inner = toStringSafe(block[2]).trim();
|
||||
const inner = toStringSafe(block.body).trim();
|
||||
|
||||
if (inner) {
|
||||
try {
|
||||
@@ -63,13 +60,13 @@ function parseMarkupSingleToolCall(block) {
|
||||
}
|
||||
}
|
||||
const input = {};
|
||||
for (const match of inner.matchAll(PARAMETER_BLOCK_PATTERN)) {
|
||||
const parameterAttrs = parseTagAttributes(match[1]);
|
||||
for (const match of findXmlElementBlocks(inner, 'parameter')) {
|
||||
const parameterAttrs = parseTagAttributes(match.attrs);
|
||||
const paramName = toStringSafe(parameterAttrs.name).trim();
|
||||
if (!paramName) {
|
||||
continue;
|
||||
}
|
||||
appendMarkupValue(input, paramName, parseMarkupValue(match[2]));
|
||||
appendMarkupValue(input, paramName, parseMarkupValue(match.body));
|
||||
}
|
||||
if (Object.keys(input).length === 0 && inner.trim() !== '') {
|
||||
return null;
|
||||
@@ -77,6 +74,154 @@ function parseMarkupSingleToolCall(block) {
|
||||
return { name, input };
|
||||
}
|
||||
|
||||
function findXmlElementBlocks(text, tag) {
|
||||
const source = toStringSafe(text);
|
||||
const name = toStringSafe(tag).toLowerCase();
|
||||
if (!source || !name) {
|
||||
return [];
|
||||
}
|
||||
const out = [];
|
||||
let pos = 0;
|
||||
while (pos < source.length) {
|
||||
const start = findXmlStartTagOutsideCDATA(source, name, pos);
|
||||
if (!start) {
|
||||
break;
|
||||
}
|
||||
const end = findMatchingXmlEndTagOutsideCDATA(source, name, start.bodyStart);
|
||||
if (!end) {
|
||||
break;
|
||||
}
|
||||
out.push({
|
||||
attrs: start.attrs,
|
||||
body: source.slice(start.bodyStart, end.closeStart),
|
||||
start: start.start,
|
||||
end: end.closeEnd,
|
||||
});
|
||||
pos = end.closeEnd;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function findXmlStartTagOutsideCDATA(text, tag, from) {
|
||||
const lower = text.toLowerCase();
|
||||
const target = `<${tag}`;
|
||||
for (let i = Math.max(0, from || 0); i < text.length;) {
|
||||
const skipped = skipXmlIgnoredSection(lower, i);
|
||||
if (skipped.blocked) {
|
||||
return null;
|
||||
}
|
||||
if (skipped.advanced) {
|
||||
i = skipped.next;
|
||||
continue;
|
||||
}
|
||||
if (lower.startsWith(target, i) && hasXmlTagBoundary(text, i + target.length)) {
|
||||
const tagEnd = findXmlTagEnd(text, i + target.length);
|
||||
if (tagEnd < 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
start: i,
|
||||
bodyStart: tagEnd + 1,
|
||||
attrs: text.slice(i + target.length, tagEnd),
|
||||
};
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findMatchingXmlEndTagOutsideCDATA(text, tag, from) {
|
||||
const lower = text.toLowerCase();
|
||||
const openTarget = `<${tag}`;
|
||||
const closeTarget = `</${tag}`;
|
||||
let depth = 1;
|
||||
for (let i = Math.max(0, from || 0); i < text.length;) {
|
||||
const skipped = skipXmlIgnoredSection(lower, i);
|
||||
if (skipped.blocked) {
|
||||
return null;
|
||||
}
|
||||
if (skipped.advanced) {
|
||||
i = skipped.next;
|
||||
continue;
|
||||
}
|
||||
if (lower.startsWith(closeTarget, i) && hasXmlTagBoundary(text, i + closeTarget.length)) {
|
||||
const tagEnd = findXmlTagEnd(text, i + closeTarget.length);
|
||||
if (tagEnd < 0) {
|
||||
return null;
|
||||
}
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
return { closeStart: i, closeEnd: tagEnd + 1 };
|
||||
}
|
||||
i = tagEnd + 1;
|
||||
continue;
|
||||
}
|
||||
if (lower.startsWith(openTarget, i) && hasXmlTagBoundary(text, i + openTarget.length)) {
|
||||
const tagEnd = findXmlTagEnd(text, i + openTarget.length);
|
||||
if (tagEnd < 0) {
|
||||
return null;
|
||||
}
|
||||
if (!isSelfClosingXmlTag(text.slice(i, tagEnd))) {
|
||||
depth += 1;
|
||||
}
|
||||
i = tagEnd + 1;
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function skipXmlIgnoredSection(lower, i) {
|
||||
if (lower.startsWith('<![cdata[', i)) {
|
||||
const end = lower.indexOf(']]>', i + '<![cdata['.length);
|
||||
if (end < 0) {
|
||||
return { advanced: false, blocked: true, next: i };
|
||||
}
|
||||
return { advanced: true, blocked: false, next: end + ']]>'.length };
|
||||
}
|
||||
if (lower.startsWith('<!--', i)) {
|
||||
const end = lower.indexOf('-->', i + '<!--'.length);
|
||||
if (end < 0) {
|
||||
return { advanced: false, blocked: true, next: i };
|
||||
}
|
||||
return { advanced: true, blocked: false, next: end + '-->'.length };
|
||||
}
|
||||
return { advanced: false, blocked: false, next: i };
|
||||
}
|
||||
|
||||
function findXmlTagEnd(text, from) {
|
||||
let quote = '';
|
||||
for (let i = Math.max(0, from || 0); i < text.length; i += 1) {
|
||||
const ch = text[i];
|
||||
if (quote) {
|
||||
if (ch === quote) {
|
||||
quote = '';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === '"' || ch === "'") {
|
||||
quote = ch;
|
||||
continue;
|
||||
}
|
||||
if (ch === '>') {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function hasXmlTagBoundary(text, idx) {
|
||||
if (idx >= text.length) {
|
||||
return true;
|
||||
}
|
||||
return [' ', '\t', '\n', '\r', '>', '/'].includes(text[idx]);
|
||||
}
|
||||
|
||||
function isSelfClosingXmlTag(startTag) {
|
||||
return toStringSafe(startTag).trim().endsWith('/');
|
||||
}
|
||||
|
||||
function parseMarkupInput(raw) {
|
||||
const s = toStringSafe(raw).trim();
|
||||
if (!s) {
|
||||
@@ -120,6 +265,10 @@ function parseMarkupKVObject(text) {
|
||||
}
|
||||
|
||||
function parseMarkupValue(raw) {
|
||||
const cdata = extractStandaloneCDATA(raw);
|
||||
if (cdata.ok) {
|
||||
return cdata.value;
|
||||
}
|
||||
const s = toStringSafe(extractRawTagValue(raw)).trim();
|
||||
if (!s) {
|
||||
return '';
|
||||
@@ -152,9 +301,9 @@ function extractRawTagValue(inner) {
|
||||
}
|
||||
|
||||
// 1. Check for CDATA
|
||||
const cdataMatch = s.match(CDATA_PATTERN);
|
||||
if (cdataMatch && cdataMatch[1] !== undefined) {
|
||||
return cdataMatch[1];
|
||||
const cdata = extractStandaloneCDATA(s);
|
||||
if (cdata.ok) {
|
||||
return cdata.value;
|
||||
}
|
||||
|
||||
// 2. Fallback to unescaping standard HTML entities
|
||||
@@ -172,6 +321,15 @@ function unescapeHtml(safe) {
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function extractStandaloneCDATA(inner) {
|
||||
const s = toStringSafe(inner).trim();
|
||||
const cdataMatch = s.match(CDATA_PATTERN);
|
||||
if (cdataMatch && cdataMatch[1] !== undefined) {
|
||||
return { ok: true, value: cdataMatch[1] };
|
||||
}
|
||||
return { ok: false, value: '' };
|
||||
}
|
||||
|
||||
function parseTagAttributes(raw) {
|
||||
const source = toStringSafe(raw);
|
||||
const out = {};
|
||||
|
||||
@@ -16,9 +16,10 @@ function consumeXMLToolCapture(captured, toolNames, trimWrappingJSONFence) {
|
||||
if (openIdx < 0) {
|
||||
continue;
|
||||
}
|
||||
// Find the LAST occurrence of the specific closing tag.
|
||||
const closeIdx = lower.lastIndexOf(pair.close);
|
||||
if (closeIdx < openIdx) {
|
||||
// Ignore closing tags that appear inside CDATA payloads, such as
|
||||
// write-file content containing tool-call documentation examples.
|
||||
const closeIdx = findXMLCloseOutsideCDATA(captured, pair.close, openIdx + pair.open.length);
|
||||
if (closeIdx < 0) {
|
||||
// Opening tag present but specific closing tag hasn't arrived.
|
||||
// Return not-ready so buffering continues until the wrapper closes.
|
||||
return { ready: false, prefix: '', calls: [], suffix: '' };
|
||||
@@ -46,8 +47,9 @@ function consumeXMLToolCapture(captured, toolNames, trimWrappingJSONFence) {
|
||||
function hasOpenXMLToolTag(captured) {
|
||||
const lower = captured.toLowerCase();
|
||||
for (const pair of XML_TOOL_TAG_PAIRS) {
|
||||
if (lower.includes(pair.open)) {
|
||||
if (!lower.includes(pair.close)) {
|
||||
const openIdx = lower.indexOf(pair.open);
|
||||
if (openIdx >= 0) {
|
||||
if (findXMLCloseOutsideCDATA(captured, pair.close, openIdx + pair.open.length) < 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -74,6 +76,38 @@ function findPartialXMLToolTagStart(s) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
function findXMLCloseOutsideCDATA(s, closeTag, start) {
|
||||
const text = typeof s === 'string' ? s : '';
|
||||
const target = String(closeTag || '').toLowerCase();
|
||||
if (!text || !target) {
|
||||
return -1;
|
||||
}
|
||||
const lower = text.toLowerCase();
|
||||
for (let i = Math.max(0, start || 0); i < text.length;) {
|
||||
if (lower.startsWith('<![cdata[', i)) {
|
||||
const end = lower.indexOf(']]>', i + '<![cdata['.length);
|
||||
if (end < 0) {
|
||||
return -1;
|
||||
}
|
||||
i = end + ']]>'.length;
|
||||
continue;
|
||||
}
|
||||
if (lower.startsWith('<!--', i)) {
|
||||
const end = lower.indexOf('-->', i + '<!--'.length);
|
||||
if (end < 0) {
|
||||
return -1;
|
||||
}
|
||||
i = end + '-->'.length;
|
||||
continue;
|
||||
}
|
||||
if (lower.startsWith(target, i)) {
|
||||
return i;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
consumeXMLToolCapture,
|
||||
hasOpenXMLToolTag,
|
||||
|
||||
@@ -43,6 +43,9 @@ func parseMarkupKVObject(text string) map[string]any {
|
||||
}
|
||||
|
||||
func parseMarkupValue(inner string) any {
|
||||
if value, ok := extractStandaloneCDATA(inner); ok {
|
||||
return value
|
||||
}
|
||||
value := strings.TrimSpace(extractRawTagValue(inner))
|
||||
if value == "" {
|
||||
return ""
|
||||
@@ -89,8 +92,8 @@ func extractRawTagValue(inner string) string {
|
||||
}
|
||||
|
||||
// 1. Check for CDATA - if present, it's the ultimate "safe" container.
|
||||
if cdataMatches := cdataPattern.FindStringSubmatch(trimmed); len(cdataMatches) >= 2 {
|
||||
return cdataMatches[1] // Return raw content between CDATA brackets
|
||||
if value, ok := extractStandaloneCDATA(trimmed); ok {
|
||||
return value // Return raw content between CDATA brackets
|
||||
}
|
||||
|
||||
// 2. If no CDATA, we still want to be robust.
|
||||
@@ -102,3 +105,11 @@ func extractRawTagValue(inner string) string {
|
||||
// but for KV objects we usually want the value.
|
||||
return html.UnescapeString(inner)
|
||||
}
|
||||
|
||||
func extractStandaloneCDATA(inner string) (string, bool) {
|
||||
trimmed := strings.TrimSpace(inner)
|
||||
if cdataMatches := cdataPattern.FindStringSubmatch(trimmed); len(cdataMatches) >= 2 {
|
||||
return cdataMatches[1], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -87,7 +87,13 @@ func stripFencedCodeBlocks(text string) string {
|
||||
lines := strings.SplitAfter(text, "\n")
|
||||
inFence := false
|
||||
fenceMarker := ""
|
||||
inCDATA := false
|
||||
for _, line := range lines {
|
||||
if inCDATA || cdataStartsBeforeFence(line) {
|
||||
b.WriteString(line)
|
||||
inCDATA = updateCDATAState(inCDATA, line)
|
||||
continue
|
||||
}
|
||||
trimmed := strings.TrimLeft(line, " \t")
|
||||
if !inFence {
|
||||
if marker, ok := parseFenceOpen(trimmed); ok {
|
||||
@@ -111,6 +117,54 @@ func stripFencedCodeBlocks(text string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func cdataStartsBeforeFence(line string) bool {
|
||||
cdataIdx := strings.Index(strings.ToLower(line), "<![cdata[")
|
||||
if cdataIdx < 0 {
|
||||
return false
|
||||
}
|
||||
fenceIdx := firstFenceMarkerIndex(line)
|
||||
return fenceIdx < 0 || cdataIdx < fenceIdx
|
||||
}
|
||||
|
||||
func firstFenceMarkerIndex(line string) int {
|
||||
idxBacktick := strings.Index(line, "```")
|
||||
idxTilde := strings.Index(line, "~~~")
|
||||
switch {
|
||||
case idxBacktick < 0:
|
||||
return idxTilde
|
||||
case idxTilde < 0:
|
||||
return idxBacktick
|
||||
case idxBacktick < idxTilde:
|
||||
return idxBacktick
|
||||
default:
|
||||
return idxTilde
|
||||
}
|
||||
}
|
||||
|
||||
func updateCDATAState(inCDATA bool, line string) bool {
|
||||
lower := strings.ToLower(line)
|
||||
pos := 0
|
||||
state := inCDATA
|
||||
for pos < len(lower) {
|
||||
if state {
|
||||
end := strings.Index(lower[pos:], "]]>")
|
||||
if end < 0 {
|
||||
return true
|
||||
}
|
||||
pos += end + len("]]>")
|
||||
state = false
|
||||
continue
|
||||
}
|
||||
start := strings.Index(lower[pos:], "<![cdata[")
|
||||
if start < 0 {
|
||||
return false
|
||||
}
|
||||
pos += start + len("<![cdata[")
|
||||
state = true
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func parseFenceOpen(line string) (string, bool) {
|
||||
if len(line) < 3 {
|
||||
return "", false
|
||||
|
||||
@@ -7,19 +7,16 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
var xmlToolCallsWrapperPattern = regexp.MustCompile(`(?is)<tool_calls\b[^>]*>\s*(.*?)\s*</tool_calls>`)
|
||||
var xmlInvokePattern = regexp.MustCompile(`(?is)<invoke\b([^>]*)>\s*(.*?)\s*</invoke>`)
|
||||
var xmlParameterPattern = regexp.MustCompile(`(?is)<parameter\b([^>]*)>\s*(.*?)\s*</parameter>`)
|
||||
var xmlAttrPattern = regexp.MustCompile(`(?is)\b([a-z0-9_:-]+)\s*=\s*("([^"]*)"|'([^']*)')`)
|
||||
var xmlToolCallsClosePattern = regexp.MustCompile(`(?is)</tool_calls>`)
|
||||
var xmlInvokeStartPattern = regexp.MustCompile(`(?is)<invoke\b[^>]*\bname\s*=\s*("([^"]*)"|'([^']*)')`)
|
||||
|
||||
func parseXMLToolCalls(text string) []ParsedToolCall {
|
||||
wrappers := xmlToolCallsWrapperPattern.FindAllStringSubmatch(text, -1)
|
||||
wrappers := findXMLElementBlocks(text, "tool_calls")
|
||||
if len(wrappers) == 0 {
|
||||
repaired := repairMissingXMLToolCallsOpeningWrapper(text)
|
||||
if repaired != text {
|
||||
wrappers = xmlToolCallsWrapperPattern.FindAllStringSubmatch(repaired, -1)
|
||||
wrappers = findXMLElementBlocks(repaired, "tool_calls")
|
||||
}
|
||||
}
|
||||
if len(wrappers) == 0 {
|
||||
@@ -27,10 +24,7 @@ func parseXMLToolCalls(text string) []ParsedToolCall {
|
||||
}
|
||||
out := make([]ParsedToolCall, 0, len(wrappers))
|
||||
for _, wrapper := range wrappers {
|
||||
if len(wrapper) < 2 {
|
||||
continue
|
||||
}
|
||||
for _, block := range xmlInvokePattern.FindAllStringSubmatch(wrapper[1], -1) {
|
||||
for _, block := range findXMLElementBlocks(wrapper.Body, "invoke") {
|
||||
call, ok := parseSingleXMLToolCall(block)
|
||||
if !ok {
|
||||
continue
|
||||
@@ -66,17 +60,14 @@ func repairMissingXMLToolCallsOpeningWrapper(text string) string {
|
||||
return text[:invokeLoc[0]] + "<tool_calls>" + text[invokeLoc[0]:closeLoc[0]] + "</tool_calls>" + text[closeLoc[1]:]
|
||||
}
|
||||
|
||||
func parseSingleXMLToolCall(block []string) (ParsedToolCall, bool) {
|
||||
if len(block) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
attrs := parseXMLTagAttributes(block[1])
|
||||
func parseSingleXMLToolCall(block xmlElementBlock) (ParsedToolCall, bool) {
|
||||
attrs := parseXMLTagAttributes(block.Attrs)
|
||||
name := strings.TrimSpace(html.UnescapeString(attrs["name"]))
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
|
||||
inner := strings.TrimSpace(block[2])
|
||||
inner := strings.TrimSpace(block.Body)
|
||||
if strings.HasPrefix(inner, "{") {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal([]byte(inner), &payload); err == nil {
|
||||
@@ -94,16 +85,13 @@ func parseSingleXMLToolCall(block []string) (ParsedToolCall, bool) {
|
||||
}
|
||||
|
||||
input := map[string]any{}
|
||||
for _, paramMatch := range xmlParameterPattern.FindAllStringSubmatch(inner, -1) {
|
||||
if len(paramMatch) < 3 {
|
||||
continue
|
||||
}
|
||||
paramAttrs := parseXMLTagAttributes(paramMatch[1])
|
||||
for _, paramMatch := range findXMLElementBlocks(inner, "parameter") {
|
||||
paramAttrs := parseXMLTagAttributes(paramMatch.Attrs)
|
||||
paramName := strings.TrimSpace(html.UnescapeString(paramAttrs["name"]))
|
||||
if paramName == "" {
|
||||
continue
|
||||
}
|
||||
value := parseInvokeParameterValue(paramMatch[2])
|
||||
value := parseInvokeParameterValue(paramMatch.Body)
|
||||
appendMarkupValue(input, paramName, value)
|
||||
}
|
||||
|
||||
@@ -116,6 +104,168 @@ func parseSingleXMLToolCall(block []string) (ParsedToolCall, bool) {
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
type xmlElementBlock struct {
|
||||
Attrs string
|
||||
Body string
|
||||
Start int
|
||||
End int
|
||||
}
|
||||
|
||||
func findXMLElementBlocks(text, tag string) []xmlElementBlock {
|
||||
if text == "" || tag == "" {
|
||||
return nil
|
||||
}
|
||||
var out []xmlElementBlock
|
||||
pos := 0
|
||||
for pos < len(text) {
|
||||
start, bodyStart, attrs, ok := findXMLStartTagOutsideCDATA(text, tag, pos)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
closeStart, closeEnd, ok := findMatchingXMLEndTagOutsideCDATA(text, tag, bodyStart)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
out = append(out, xmlElementBlock{
|
||||
Attrs: attrs,
|
||||
Body: text[bodyStart:closeStart],
|
||||
Start: start,
|
||||
End: closeEnd,
|
||||
})
|
||||
pos = closeEnd
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func findXMLStartTagOutsideCDATA(text, tag string, from int) (start, bodyStart int, attrs string, ok bool) {
|
||||
lower := strings.ToLower(text)
|
||||
target := "<" + strings.ToLower(tag)
|
||||
for i := maxInt(from, 0); i < len(text); {
|
||||
next, advanced, blocked := skipXMLIgnoredSection(lower, i)
|
||||
if blocked {
|
||||
return -1, -1, "", false
|
||||
}
|
||||
if advanced {
|
||||
i = next
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(lower[i:], target) && hasXMLTagBoundary(text, i+len(target)) {
|
||||
end := findXMLTagEnd(text, i+len(target))
|
||||
if end < 0 {
|
||||
return -1, -1, "", false
|
||||
}
|
||||
return i, end + 1, text[i+len(target) : end], true
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1, -1, "", false
|
||||
}
|
||||
|
||||
func findMatchingXMLEndTagOutsideCDATA(text, tag string, from int) (closeStart, closeEnd int, ok bool) {
|
||||
lower := strings.ToLower(text)
|
||||
openTarget := "<" + strings.ToLower(tag)
|
||||
closeTarget := "</" + strings.ToLower(tag)
|
||||
depth := 1
|
||||
for i := maxInt(from, 0); i < len(text); {
|
||||
next, advanced, blocked := skipXMLIgnoredSection(lower, i)
|
||||
if blocked {
|
||||
return -1, -1, false
|
||||
}
|
||||
if advanced {
|
||||
i = next
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(lower[i:], closeTarget) && hasXMLTagBoundary(text, i+len(closeTarget)) {
|
||||
end := findXMLTagEnd(text, i+len(closeTarget))
|
||||
if end < 0 {
|
||||
return -1, -1, false
|
||||
}
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i, end + 1, true
|
||||
}
|
||||
i = end + 1
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(lower[i:], openTarget) && hasXMLTagBoundary(text, i+len(openTarget)) {
|
||||
end := findXMLTagEnd(text, i+len(openTarget))
|
||||
if end < 0 {
|
||||
return -1, -1, false
|
||||
}
|
||||
if !isSelfClosingXMLTag(text[:end]) {
|
||||
depth++
|
||||
}
|
||||
i = end + 1
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1, -1, false
|
||||
}
|
||||
|
||||
func skipXMLIgnoredSection(lower string, i int) (next int, advanced bool, blocked bool) {
|
||||
switch {
|
||||
case strings.HasPrefix(lower[i:], "<![cdata["):
|
||||
end := strings.Index(lower[i+len("<![cdata["):], "]]>")
|
||||
if end < 0 {
|
||||
return 0, false, true
|
||||
}
|
||||
return i + len("<![cdata[") + end + len("]]>"), true, false
|
||||
case strings.HasPrefix(lower[i:], "<!--"):
|
||||
end := strings.Index(lower[i+len("<!--"):], "-->")
|
||||
if end < 0 {
|
||||
return 0, false, true
|
||||
}
|
||||
return i + len("<!--") + end + len("-->"), true, false
|
||||
default:
|
||||
return i, false, false
|
||||
}
|
||||
}
|
||||
|
||||
func findXMLTagEnd(text string, from int) int {
|
||||
quote := byte(0)
|
||||
for i := maxInt(from, 0); i < len(text); i++ {
|
||||
ch := text[i]
|
||||
if quote != 0 {
|
||||
if ch == quote {
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ch == '"' || ch == '\'' {
|
||||
quote = ch
|
||||
continue
|
||||
}
|
||||
if ch == '>' {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func hasXMLTagBoundary(text string, idx int) bool {
|
||||
if idx >= len(text) {
|
||||
return true
|
||||
}
|
||||
switch text[idx] {
|
||||
case ' ', '\t', '\n', '\r', '>', '/':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isSelfClosingXMLTag(startTag string) bool {
|
||||
return strings.HasSuffix(strings.TrimSpace(startTag), "/")
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func parseXMLTagAttributes(raw string) map[string]string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return map[string]string{}
|
||||
@@ -143,6 +293,9 @@ func parseInvokeParameterValue(raw string) any {
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
if value, ok := extractStandaloneCDATA(trimmed); ok {
|
||||
return value
|
||||
}
|
||||
if parsed := parseStructuredToolCallInput(trimmed); len(parsed) > 0 {
|
||||
if len(parsed) == 1 {
|
||||
if rawValue, ok := parsed["_raw"].(string); ok {
|
||||
|
||||
@@ -54,6 +54,32 @@ echo "hello"
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsKeepsToolSyntaxInsideCDATAAsParameterText(t *testing.T) {
|
||||
payload := strings.Join([]string{
|
||||
"# Release notes",
|
||||
"",
|
||||
"```xml",
|
||||
"<tool_calls>",
|
||||
" <invoke name=\"demo\">",
|
||||
" <parameter name=\"value\">x</parameter>",
|
||||
" </invoke>",
|
||||
"</tool_calls>",
|
||||
"```",
|
||||
}, "\n")
|
||||
text := `<tool_calls><invoke name="Write"><parameter name="content"><![CDATA[` + payload + `]]></parameter><parameter name="file_path">DS2API-4.0-Release-Notes.md</parameter></invoke></tool_calls>`
|
||||
calls := ParseToolCalls(text, []string{"Write"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
content, _ := calls[0].Input["content"].(string)
|
||||
if content != payload {
|
||||
t.Fatalf("expected CDATA payload with nested tool syntax to survive intact, got %q", content)
|
||||
}
|
||||
if calls[0].Input["file_path"] != "DS2API-4.0-Release-Notes.md" {
|
||||
t.Fatalf("expected file_path parameter, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsInvokeParameters(t *testing.T) {
|
||||
text := `<tool_calls><invoke name="get_weather"><parameter name="city">beijing</parameter><parameter name="unit">c</parameter></invoke></tool_calls>`
|
||||
calls := ParseToolCalls(text, []string{"get_weather"})
|
||||
|
||||
@@ -35,9 +35,10 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
|
||||
if openIdx < 0 {
|
||||
continue
|
||||
}
|
||||
// Find the LAST occurrence of the specific closing tag to get the outermost block.
|
||||
closeIdx := strings.LastIndex(lower, pair.close)
|
||||
if closeIdx < openIdx {
|
||||
// Find the matching closing tag outside CDATA. Long write-file tool
|
||||
// calls often contain XML examples in CDATA, including </tool_calls>.
|
||||
closeIdx := findXMLCloseOutsideCDATA(captured, pair.close, openIdx+len(pair.open))
|
||||
if closeIdx < 0 {
|
||||
// Opening tag is present but its specific closing tag hasn't arrived.
|
||||
// Return not-ready so we keep buffering until the canonical wrapper closes.
|
||||
return "", nil, "", false
|
||||
@@ -57,7 +58,7 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
|
||||
}
|
||||
if !strings.Contains(lower, "<tool_calls") {
|
||||
invokeIdx := strings.Index(lower, "<invoke")
|
||||
closeIdx := strings.LastIndex(lower, "</tool_calls>")
|
||||
closeIdx := findXMLCloseOutsideCDATA(captured, "</tool_calls>", invokeIdx)
|
||||
if invokeIdx >= 0 && closeIdx > invokeIdx {
|
||||
closeEnd := closeIdx + len("</tool_calls>")
|
||||
xmlBlock := "<tool_calls>" + captured[invokeIdx:closeIdx] + "</tool_calls>"
|
||||
@@ -79,8 +80,9 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
|
||||
func hasOpenXMLToolTag(captured string) bool {
|
||||
lower := strings.ToLower(captured)
|
||||
for _, pair := range xmlToolCallTagPairs {
|
||||
if strings.Contains(lower, pair.open) {
|
||||
if !strings.Contains(lower, pair.close) {
|
||||
openIdx := strings.Index(lower, pair.open)
|
||||
if openIdx >= 0 {
|
||||
if findXMLCloseOutsideCDATA(captured, pair.close, openIdx+len(pair.open)) < 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -88,6 +90,38 @@ func hasOpenXMLToolTag(captured string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func findXMLCloseOutsideCDATA(s, closeTag string, start int) int {
|
||||
if s == "" || closeTag == "" {
|
||||
return -1
|
||||
}
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
lower := strings.ToLower(s)
|
||||
target := strings.ToLower(closeTag)
|
||||
for i := start; i < len(s); {
|
||||
switch {
|
||||
case strings.HasPrefix(lower[i:], "<![cdata["):
|
||||
end := strings.Index(lower[i+len("<![cdata["):], "]]>")
|
||||
if end < 0 {
|
||||
return -1
|
||||
}
|
||||
i += len("<![cdata[") + end + len("]]>")
|
||||
case strings.HasPrefix(lower[i:], "<!--"):
|
||||
end := strings.Index(lower[i+len("<!--"):], "-->")
|
||||
if end < 0 {
|
||||
return -1
|
||||
}
|
||||
i += len("<!--") + end + len("-->")
|
||||
case strings.HasPrefix(lower[i:], target):
|
||||
return i
|
||||
default:
|
||||
i++
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// findPartialXMLToolTagStart checks if the string ends with a partial canonical
|
||||
// XML wrapper tag (e.g., "<too") and returns the position of the '<'.
|
||||
func findPartialXMLToolTagStart(s string) int {
|
||||
|
||||
@@ -84,6 +84,65 @@ func TestProcessToolSieveHandlesLongXMLToolCall(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessToolSieveKeepsCDATAEmbeddedToolClosingBuffered(t *testing.T) {
|
||||
var state State
|
||||
payload := strings.Join([]string{
|
||||
"# DS2API 4.0 更新内容",
|
||||
"",
|
||||
strings.Repeat("x", 4096),
|
||||
"```xml",
|
||||
"<tool_calls>",
|
||||
" <invoke name=\"demo\">",
|
||||
" <parameter name=\"value\">x</parameter>",
|
||||
" </invoke>",
|
||||
"</tool_calls>",
|
||||
"```",
|
||||
"tail",
|
||||
}, "\n")
|
||||
innerClose := strings.Index(payload, "</tool_calls>") + len("</tool_calls>")
|
||||
chunks := []string{
|
||||
"<tool_calls>\n <invoke name=\"Write\">\n <parameter name=\"content\"><![CDATA[",
|
||||
payload[:innerClose],
|
||||
payload[innerClose:],
|
||||
"]]></parameter>\n <parameter name=\"file_path\">DS2API-4.0-Release-Notes.md</parameter>\n </invoke>\n</tool_calls>",
|
||||
}
|
||||
|
||||
var events []Event
|
||||
for i, c := range chunks {
|
||||
next := ProcessChunk(&state, c, []string{"Write"})
|
||||
if i <= 1 {
|
||||
for _, evt := range next {
|
||||
if evt.Content != "" || len(evt.ToolCalls) > 0 {
|
||||
t.Fatalf("expected no events before outer closing tag, chunk=%d events=%#v", i, next)
|
||||
}
|
||||
}
|
||||
}
|
||||
events = append(events, next...)
|
||||
}
|
||||
events = append(events, Flush(&state, []string{"Write"})...)
|
||||
|
||||
var textContent strings.Builder
|
||||
var gotPayload string
|
||||
toolCalls := 0
|
||||
for _, evt := range events {
|
||||
textContent.WriteString(evt.Content)
|
||||
if len(evt.ToolCalls) > 0 {
|
||||
toolCalls += len(evt.ToolCalls)
|
||||
gotPayload, _ = evt.ToolCalls[0].Input["content"].(string)
|
||||
}
|
||||
}
|
||||
|
||||
if toolCalls != 1 {
|
||||
t.Fatalf("expected one parsed tool call, got %d events=%#v", toolCalls, events)
|
||||
}
|
||||
if textContent.Len() != 0 {
|
||||
t.Fatalf("expected no leaked text, got %q", textContent.String())
|
||||
}
|
||||
if gotPayload != payload {
|
||||
t.Fatalf("expected full CDATA payload to survive intact, got len=%d want=%d", len(gotPayload), len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessToolSieveXMLWithLeadingText(t *testing.T) {
|
||||
var state State
|
||||
// Model outputs some prose then an XML tool call.
|
||||
|
||||
@@ -118,6 +118,60 @@ test('sieve keeps long XML tool calls buffered until the closing tag arrives', (
|
||||
assert.equal(finalCalls[0].input.content, longContent);
|
||||
});
|
||||
|
||||
test('sieve keeps CDATA tool examples buffered until the outer closing tag arrives', () => {
|
||||
const content = [
|
||||
'# DS2API 4.0 更新内容',
|
||||
'',
|
||||
'x'.repeat(4096),
|
||||
'```xml',
|
||||
'<tool_calls>',
|
||||
' <invoke name="demo">',
|
||||
' <parameter name="value">x</parameter>',
|
||||
' </invoke>',
|
||||
'</tool_calls>',
|
||||
'```',
|
||||
'tail',
|
||||
].join('\n');
|
||||
const innerClose = content.indexOf('</tool_calls>') + '</tool_calls>'.length;
|
||||
const state = createToolSieveState();
|
||||
const chunks = [
|
||||
'<tool_calls>\n <invoke name="Write">\n <parameter name="content"><![CDATA[',
|
||||
content.slice(0, innerClose),
|
||||
content.slice(innerClose),
|
||||
']]></parameter>\n <parameter name="file_path">DS2API-4.0-Release-Notes.md</parameter>\n </invoke>\n</tool_calls>',
|
||||
];
|
||||
const events = [];
|
||||
chunks.forEach((chunk, idx) => {
|
||||
const next = processToolSieveChunk(state, chunk, ['Write']);
|
||||
if (idx <= 1) {
|
||||
assert.deepEqual(next, []);
|
||||
}
|
||||
events.push(...next);
|
||||
});
|
||||
events.push(...flushToolSieve(state, ['Write']));
|
||||
|
||||
const leakedText = collectText(events);
|
||||
const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []);
|
||||
assert.equal(leakedText, '');
|
||||
assert.equal(finalCalls.length, 1);
|
||||
assert.equal(finalCalls[0].name, 'Write');
|
||||
assert.equal(finalCalls[0].input.content, content);
|
||||
});
|
||||
|
||||
test('parseToolCalls keeps XML-looking CDATA content intact', () => {
|
||||
const content = [
|
||||
'# Release notes',
|
||||
'```xml',
|
||||
'<tool_calls><invoke name="demo"><parameter name="value">x</parameter></invoke></tool_calls>',
|
||||
'```',
|
||||
].join('\n');
|
||||
const payload = `<tool_calls><invoke name="Write"><parameter name="content"><![CDATA[${content}]]></parameter><parameter name="file_path">DS2API-4.0-Release-Notes.md</parameter></invoke></tool_calls>`;
|
||||
const calls = parseToolCalls(payload, ['Write']);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].input.content, content);
|
||||
assert.equal(calls[0].input.file_path, 'DS2API-4.0-Release-Notes.md');
|
||||
});
|
||||
|
||||
test('sieve passes JSON tool_calls payload through as text (XML-only)', () => {
|
||||
const events = runSieve(
|
||||
['{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}'],
|
||||
|
||||
Reference in New Issue
Block a user