From 302bcefeb52cb718421955e27c4efe2e55b57581 Mon Sep 17 00:00:00 2001 From: CJACK Date: Sun, 29 Mar 2026 13:01:11 +0800 Subject: [PATCH] feat: implement XML-based tool call extraction and refactor sieve utilities into dedicated modules --- internal/adapter/openai/tool_sieve_core.go | 92 ++++------- .../adapter/openai/tool_sieve_jsonscan.go | 66 ++++++++ internal/adapter/openai/tool_sieve_xml.go | 109 ++++++++++++ .../adapter/openai/tool_sieve_xml_test.go | 155 ++++++++++++++++++ .../js/helpers/stream-tool-sieve/jsonscan.js | 49 ++++++ .../js/helpers/stream-tool-sieve/sieve-xml.js | 91 ++++++++++ .../js/helpers/stream-tool-sieve/sieve.js | 94 +++++------ .../stream-tool-sieve/tool-keywords.js | 16 ++ plans/node-syntax-gate-targets.txt | 2 + plans/refactor-line-gate-targets.txt | 2 + 10 files changed, 563 insertions(+), 113 deletions(-) create mode 100644 internal/adapter/openai/tool_sieve_xml.go create mode 100644 internal/adapter/openai/tool_sieve_xml_test.go create mode 100644 internal/js/helpers/stream-tool-sieve/sieve-xml.js diff --git a/internal/adapter/openai/tool_sieve_core.go b/internal/adapter/openai/tool_sieve_core.go index ad2c231..e651445 100644 --- a/internal/adapter/openai/tool_sieve_core.go +++ b/internal/adapter/openai/tool_sieve_core.go @@ -159,6 +159,10 @@ func findSuspiciousPrefixStart(s string) int { start = idx } } + // Also check for partial XML tool tag at end of string. + if xmlIdx := findPartialXMLToolTagStart(s); xmlIdx >= 0 && xmlIdx > start { + start = xmlIdx + } return start } @@ -175,9 +179,23 @@ func findToolSegmentStart(s string) int { bestKeyIdx = idx } } + // Also detect XML tool call tags. + for _, tag := range xmlToolTagsToDetect { + idx := strings.Index(lower, tag) + if idx >= 0 && (bestKeyIdx < 0 || idx < bestKeyIdx) { + bestKeyIdx = idx + } + } if bestKeyIdx < 0 { return -1 } + // For XML tags, the '<' is itself the segment start. + if bestKeyIdx < len(s) && s[bestKeyIdx] == '<' { + if fenceStart, ok := openFenceStartBefore(s, bestKeyIdx); ok { + return fenceStart + } + return bestKeyIdx + } start := strings.LastIndex(s[:bestKeyIdx], "{") if start < 0 { start = bestKeyIdx @@ -193,6 +211,16 @@ func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix if captured == "" { return "", nil, "", false } + + // Try XML tool call extraction first. + if xmlPrefix, xmlCalls, xmlSuffix, xmlReady := consumeXMLToolCapture(captured, toolNames); xmlReady { + return xmlPrefix, xmlCalls, xmlSuffix, true + } + // If XML tags are present but block is incomplete, keep buffering. + if hasOpenXMLToolTag(captured) { + return "", nil, "", false + } + lower := strings.ToLower(captured) keyIdx := -1 keywords := []string{"tool_calls", "\"function\"", "function.name:", "[tool_call_history]", "[tool_result_history]"} @@ -234,67 +262,3 @@ func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart) return prefixPart, parsed.Calls, suffixPart, true } - -func extractToolHistoryBlock(captured string, keyIdx int) (start int, end int, ok bool) { - if keyIdx < 0 || keyIdx >= len(captured) { - return 0, 0, false - } - rest := strings.ToLower(captured[keyIdx:]) - switch { - case strings.HasPrefix(rest, "[tool_call_history]"): - closeTag := "[/tool_call_history]" - closeIdx := strings.Index(rest, closeTag) - if closeIdx < 0 { - return 0, 0, false - } - return keyIdx, keyIdx + closeIdx + len(closeTag), true - case strings.HasPrefix(rest, "[tool_result_history]"): - closeTag := "[/tool_result_history]" - closeIdx := strings.Index(rest, closeTag) - if closeIdx < 0 { - return 0, 0, false - } - return keyIdx, keyIdx + closeIdx + len(closeTag), true - default: - return 0, 0, false - } -} - -func trimWrappingJSONFence(prefix, suffix string) (string, string) { - trimmedPrefix := strings.TrimRight(prefix, " \t\r\n") - fenceIdx := strings.LastIndex(trimmedPrefix, "```") - if fenceIdx < 0 { - return prefix, suffix - } - // Only strip when the trailing fence in prefix behaves like an opening fence. - // A legitimate closing fence before a standalone tool JSON must be preserved. - if strings.Count(trimmedPrefix[:fenceIdx+3], "```")%2 == 0 { - return prefix, suffix - } - fenceHeader := strings.TrimSpace(trimmedPrefix[fenceIdx+3:]) - if fenceHeader != "" && !strings.EqualFold(fenceHeader, "json") { - return prefix, suffix - } - - trimmedSuffix := strings.TrimLeft(suffix, " \t\r\n") - if !strings.HasPrefix(trimmedSuffix, "```") { - return prefix, suffix - } - consumedLeading := len(suffix) - len(trimmedSuffix) - return trimmedPrefix[:fenceIdx], suffix[consumedLeading+3:] -} - -func openFenceStartBefore(s string, pos int) (int, bool) { - if pos <= 0 || pos > len(s) { - return -1, false - } - segment := s[:pos] - lastFence := strings.LastIndex(segment, "```") - if lastFence < 0 { - return -1, false - } - if strings.Count(segment, "```")%2 == 1 { - return lastFence, true - } - return -1, false -} diff --git a/internal/adapter/openai/tool_sieve_jsonscan.go b/internal/adapter/openai/tool_sieve_jsonscan.go index b49ef7a..deb745a 100644 --- a/internal/adapter/openai/tool_sieve_jsonscan.go +++ b/internal/adapter/openai/tool_sieve_jsonscan.go @@ -1,5 +1,7 @@ package openai +import "strings" + func extractJSONObjectFrom(text string, start int) (string, int, bool) { if start < 0 || start >= len(text) || text[start] != '{' { return "", 0, false @@ -41,3 +43,67 @@ func extractJSONObjectFrom(text string, start int) (string, int, bool) { } return "", 0, false } + +func extractToolHistoryBlock(captured string, keyIdx int) (start int, end int, ok bool) { + if keyIdx < 0 || keyIdx >= len(captured) { + return 0, 0, false + } + rest := strings.ToLower(captured[keyIdx:]) + switch { + case strings.HasPrefix(rest, "[tool_call_history]"): + closeTag := "[/tool_call_history]" + closeIdx := strings.Index(rest, closeTag) + if closeIdx < 0 { + return 0, 0, false + } + return keyIdx, keyIdx + closeIdx + len(closeTag), true + case strings.HasPrefix(rest, "[tool_result_history]"): + closeTag := "[/tool_result_history]" + closeIdx := strings.Index(rest, closeTag) + if closeIdx < 0 { + return 0, 0, false + } + return keyIdx, keyIdx + closeIdx + len(closeTag), true + default: + return 0, 0, false + } +} + +func trimWrappingJSONFence(prefix, suffix string) (string, string) { + trimmedPrefix := strings.TrimRight(prefix, " \t\r\n") + fenceIdx := strings.LastIndex(trimmedPrefix, "```") + if fenceIdx < 0 { + return prefix, suffix + } + // Only strip when the trailing fence in prefix behaves like an opening fence. + // A legitimate closing fence before a standalone tool JSON must be preserved. + if strings.Count(trimmedPrefix[:fenceIdx+3], "```")%2 == 0 { + return prefix, suffix + } + fenceHeader := strings.TrimSpace(trimmedPrefix[fenceIdx+3:]) + if fenceHeader != "" && !strings.EqualFold(fenceHeader, "json") { + return prefix, suffix + } + + trimmedSuffix := strings.TrimLeft(suffix, " \t\r\n") + if !strings.HasPrefix(trimmedSuffix, "```") { + return prefix, suffix + } + consumedLeading := len(suffix) - len(trimmedSuffix) + return trimmedPrefix[:fenceIdx], suffix[consumedLeading+3:] +} + +func openFenceStartBefore(s string, pos int) (int, bool) { + if pos <= 0 || pos > len(s) { + return -1, false + } + segment := s[:pos] + lastFence := strings.LastIndex(segment, "```") + if lastFence < 0 { + return -1, false + } + if strings.Count(segment, "```")%2 == 1 { + return lastFence, true + } + return -1, false +} diff --git a/internal/adapter/openai/tool_sieve_xml.go b/internal/adapter/openai/tool_sieve_xml.go new file mode 100644 index 0000000..885f50a --- /dev/null +++ b/internal/adapter/openai/tool_sieve_xml.go @@ -0,0 +1,109 @@ +package openai + +import ( + "regexp" + "strings" + + "ds2api/internal/util" +) + +// --- XML tool call support for the streaming sieve --- + +var xmlToolCallClosingTags = []string{"", "", "", "", "", ""} +var xmlToolCallOpeningTags = []string{"\s*(?:.*?)\s*|\s*(?:.*?)\s*|]*>(?:.*?)|]*>(?:.*?)|(?:.*?))`) + +// xmlToolTagsToDetect is the set of XML tag prefixes used by findToolSegmentStart. +var xmlToolTagsToDetect = []string{"", "", "", ""} + +// consumeXMLToolCapture tries to extract complete XML tool call blocks from captured text. +func consumeXMLToolCapture(captured string, toolNames []string) (prefix string, calls []util.ParsedToolCall, suffix string, ready bool) { + lower := strings.ToLower(captured) + // Find the earliest XML tool opening tag. + openIdx := -1 + for _, tag := range xmlToolCallOpeningTags { + idx := strings.Index(lower, tag) + if idx >= 0 && (openIdx < 0 || idx < openIdx) { + openIdx = idx + } + } + if openIdx < 0 { + return "", nil, "", false + } + + // Look for a matching closing tag. + closeIdx := -1 + for _, tag := range xmlToolCallClosingTags { + idx := strings.Index(lower[openIdx:], tag) + if idx >= 0 { + absEnd := openIdx + idx + len(tag) + if closeIdx < 0 || absEnd > closeIdx { + closeIdx = absEnd + } + } + } + if closeIdx <= 0 { + return "", nil, "", false + } + + xmlBlock := captured[openIdx:closeIdx] + prefixPart := captured[:openIdx] + suffixPart := captured[closeIdx:] + parsed := util.ParseToolCalls(xmlBlock, toolNames) + if len(parsed) > 0 { + prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart) + return prefixPart, parsed, suffixPart, true + } + // Looks like XML tool syntax but failed to parse — consume it to avoid leak. + return prefixPart, nil, suffixPart, true +} + +// hasOpenXMLToolTag returns true if captured text contains an XML tool opening tag +// but no corresponding closing tag yet. +func hasOpenXMLToolTag(captured string) bool { + lower := strings.ToLower(captured) + for _, tag := range xmlToolCallOpeningTags { + if strings.Contains(lower, tag) { + hasClosed := false + for _, ct := range xmlToolCallClosingTags { + if strings.Contains(lower, ct) { + hasClosed = true + break + } + } + if !hasClosed { + return true + } + } + } + return false +} + +// findPartialXMLToolTagStart checks if the string ends with a partial XML tool tag +// (e.g., "' in the tail, the tag is closed — not partial. + if strings.Contains(tail, ">") { + return -1 + } + lowerTail := strings.ToLower(tail) + // Check if the tail is a prefix of any known XML tool tag. + for _, tag := range xmlToolCallOpeningTags { + tagWithLT := tag + if !strings.HasPrefix(tagWithLT, "<") { + tagWithLT = "<" + tagWithLT + } + if strings.HasPrefix(tagWithLT, lowerTail) { + return lastLT + } + } + return -1 +} diff --git a/internal/adapter/openai/tool_sieve_xml_test.go b/internal/adapter/openai/tool_sieve_xml_test.go new file mode 100644 index 0000000..b04f87b --- /dev/null +++ b/internal/adapter/openai/tool_sieve_xml_test.go @@ -0,0 +1,155 @@ +package openai + +import ( + "strings" + "testing" +) + +func TestProcessToolSieveInterceptsXMLToolCallWithoutLeak(t *testing.T) { + var state toolStreamSieveState + // Simulate a model producing XML tool call output chunk by chunk. + chunks := []string{ + "\n", + " \n", + " read_file\n", + ` {"path":"README.MD"}` + "\n", + " \n", + "", + } + var events []toolStreamEvent + for _, c := range chunks { + events = append(events, processToolSieveChunk(&state, c, []string{"read_file"})...) + } + events = append(events, flushToolSieve(&state, []string{"read_file"})...) + + var textContent string + var toolCalls int + for _, evt := range events { + if evt.Content != "" { + textContent += evt.Content + } + toolCalls += len(evt.ToolCalls) + } + + if strings.Contains(textContent, "\n \n read_file\n", + ` {"path":"go.mod"}` + "\n \n", + } + var events []toolStreamEvent + for _, c := range chunks { + events = append(events, processToolSieveChunk(&state, c, []string{"read_file"})...) + } + events = append(events, flushToolSieve(&state, []string{"read_file"})...) + + var textContent string + var toolCalls int + for _, evt := range events { + if evt.Content != "" { + textContent += evt.Content + } + toolCalls += len(evt.ToolCalls) + } + + // Leading text should be emitted. + if !strings.Contains(textContent, "Let me check the file.") { + t.Fatalf("expected leading text to be emitted, got %q", textContent) + } + // The XML itself should NOT leak. + if strings.Contains(textContent, "\n", 10}, + {"tool_call_tag", "prefix \n", 7}, + {"invoke_tag", "text body", 5}, + {"function_call_tag", "body", 0}, + {"no_xml", "just plain text", -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := findToolSegmentStart(tc.input) + if got != tc.want { + t.Fatalf("findToolSegmentStart(%q) = %d, want %d", tc.input, got, tc.want) + } + }) + } +} + +func TestFindPartialXMLToolTagStart(t *testing.T) { + cases := []struct { + name string + input string + want int + }{ + {"partial_tool_call", "Hello done", -1}, + {"no_lt", "plain text", -1}, + {"closed_lt", "a < b > c", -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := findPartialXMLToolTagStart(tc.input) + if got != tc.want { + t.Fatalf("findPartialXMLToolTagStart(%q) = %d, want %d", tc.input, got, tc.want) + } + }) + } +} + +func TestHasOpenXMLToolTag(t *testing.T) { + if !hasOpenXMLToolTag("\nfoo") { + t.Fatal("should detect open XML tool tag without closing tag") + } + if hasOpenXMLToolTag("\nfoo") { + t.Fatal("should return false when closing tag is present") + } + if hasOpenXMLToolTag("plain text without any XML") { + t.Fatal("should return false for plain text") + } +} diff --git a/internal/js/helpers/stream-tool-sieve/jsonscan.js b/internal/js/helpers/stream-tool-sieve/jsonscan.js index a86ed05..4f68d92 100644 --- a/internal/js/helpers/stream-tool-sieve/jsonscan.js +++ b/internal/js/helpers/stream-tool-sieve/jsonscan.js @@ -140,9 +140,58 @@ function extractJSONObjectFrom(text, start) { return { ok: false, end: 0 }; } +function extractToolHistoryBlock(captured, keyIdx) { + if (typeof captured !== 'string' || keyIdx < 0 || keyIdx >= captured.length) { + return { ok: false, start: 0, end: 0 }; + } + const rest = captured.slice(keyIdx).toLowerCase(); + if (rest.startsWith('[tool_call_history]')) { + const closeTag = '[/tool_call_history]'; + const closeIdx = rest.indexOf(closeTag); + if (closeIdx < 0) { + return { ok: false, start: 0, end: 0 }; + } + return { ok: true, start: keyIdx, end: keyIdx + closeIdx + closeTag.length }; + } + if (rest.startsWith('[tool_result_history]')) { + const closeTag = '[/tool_result_history]'; + const closeIdx = rest.indexOf(closeTag); + if (closeIdx < 0) { + return { ok: false, start: 0, end: 0 }; + } + return { ok: true, start: keyIdx, end: keyIdx + closeIdx + closeTag.length }; + } + return { ok: false, start: 0, end: 0 }; +} + +function trimWrappingJSONFence(prefix, suffix) { + const rightTrimmedPrefix = (prefix || '').replace(/[ \t\r\n]+$/g, ''); + const fenceIdx = rightTrimmedPrefix.lastIndexOf('```'); + if (fenceIdx < 0) return { prefix, suffix }; + const fenceCount = (rightTrimmedPrefix.slice(0, fenceIdx + 3).match(/```/g) || []).length; + if (fenceCount % 2 === 0) { + return { prefix, suffix }; + } + const header = rightTrimmedPrefix.slice(fenceIdx + 3).trim().toLowerCase(); + if (header && header !== 'json') { + return { prefix, suffix }; + } + const leftTrimmedSuffix = (suffix || '').replace(/^[ \t\r\n]+/g, ''); + if (!leftTrimmedSuffix.startsWith('```')) { + return { prefix, suffix }; + } + const consumed = (suffix || '').length - leftTrimmedSuffix.length; + return { + prefix: rightTrimmedPrefix.slice(0, fenceIdx), + suffix: (suffix || '').slice(consumed + 3), + }; +} + module.exports = { findObjectFieldValueStart, parseJSONStringLiteral, skipSpaces, extractJSONObjectFrom, + extractToolHistoryBlock, + trimWrappingJSONFence, }; diff --git a/internal/js/helpers/stream-tool-sieve/sieve-xml.js b/internal/js/helpers/stream-tool-sieve/sieve-xml.js new file mode 100644 index 0000000..c96ff60 --- /dev/null +++ b/internal/js/helpers/stream-tool-sieve/sieve-xml.js @@ -0,0 +1,91 @@ +'use strict'; +const { parseToolCalls } = require('./parse'); +const { + XML_TOOL_OPENING_TAGS, + XML_TOOL_CLOSING_TAGS, +} = require('./tool-keywords'); + +function consumeXMLToolCapture(captured, toolNames, trimWrappingJSONFence) { + const lower = captured.toLowerCase(); + let openIdx = -1; + for (const tag of XML_TOOL_OPENING_TAGS) { + const idx = lower.indexOf(tag); + if (idx >= 0 && (openIdx < 0 || idx < openIdx)) { + openIdx = idx; + } + } + if (openIdx < 0) { + return { ready: false, prefix: '', calls: [], suffix: '' }; + } + let closeIdx = -1; + for (const tag of XML_TOOL_CLOSING_TAGS) { + const idx = lower.indexOf(tag, openIdx); + if (idx >= 0) { + const absEnd = idx + tag.length; + if (closeIdx < 0 || absEnd > closeIdx) { + closeIdx = absEnd; + } + } + } + if (closeIdx <= 0) { + return { ready: false, prefix: '', calls: [], suffix: '' }; + } + const xmlBlock = captured.slice(openIdx, closeIdx); + let prefixPart = captured.slice(0, openIdx); + let suffixPart = captured.slice(closeIdx); + const parsed = parseToolCalls(xmlBlock, toolNames); + if (Array.isArray(parsed) && parsed.length > 0) { + const trimmedFence = trimWrappingJSONFence(prefixPart, suffixPart); + return { + ready: true, + prefix: trimmedFence.prefix, + calls: parsed, + suffix: trimmedFence.suffix, + }; + } + return { ready: true, prefix: prefixPart, calls: [], suffix: suffixPart }; +} + +function hasOpenXMLToolTag(captured) { + const lower = captured.toLowerCase(); + for (const tag of XML_TOOL_OPENING_TAGS) { + if (lower.includes(tag)) { + let hasClosed = false; + for (const ct of XML_TOOL_CLOSING_TAGS) { + if (lower.includes(ct)) { + hasClosed = true; + break; + } + } + if (!hasClosed) { + return true; + } + } + } + return false; +} + +function findPartialXMLToolTagStart(s) { + const lastLT = s.lastIndexOf('<'); + if (lastLT < 0) { + return -1; + } + const tail = s.slice(lastLT); + if (tail.includes('>')) { + return -1; + } + const lowerTail = tail.toLowerCase(); + for (const tag of XML_TOOL_OPENING_TAGS) { + const tagWithLT = tag.startsWith('<') ? tag : '<' + tag; + if (tagWithLT.startsWith(lowerTail)) { + return lastLT; + } + } + return -1; +} + +module.exports = { + consumeXMLToolCapture, + hasOpenXMLToolTag, + findPartialXMLToolTagStart, +}; diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index b930b25..bd7e7cc 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -5,8 +5,17 @@ const { insideCodeFenceWithState, } = require('./state'); const { parseStandaloneToolCallsDetailed } = require('./parse'); -const { extractJSONObjectFrom } = require('./jsonscan'); -const { TOOL_SEGMENT_KEYWORDS, earliestKeywordIndex } = require('./tool-keywords'); +const { extractJSONObjectFrom, extractToolHistoryBlock, trimWrappingJSONFence } = require('./jsonscan'); +const { + TOOL_SEGMENT_KEYWORDS, + XML_TOOL_SEGMENT_TAGS, + earliestKeywordIndex, +} = require('./tool-keywords'); +const { + consumeXMLToolCapture: consumeXMLToolCaptureImpl, + hasOpenXMLToolTag, + findPartialXMLToolTagStart, +} = require('./sieve-xml'); function processToolSieveChunk(state, chunk, toolNames) { if (!state) { return []; @@ -144,6 +153,11 @@ function findSuspiciousPrefixStart(s) { start = idx; } } + // Also check for partial XML tool tag at end of string. + const xmlIdx = findPartialXMLToolTagStart(s); + if (xmlIdx >= 0 && xmlIdx > start) { + start = xmlIdx; + } return start; } @@ -154,10 +168,27 @@ function findToolSegmentStart(state, s) { const lower = s.toLowerCase(); let offset = 0; while (true) { - const { index: bestKeyIdx, keyword: matchedKeyword } = earliestKeywordIndex(lower, TOOL_SEGMENT_KEYWORDS, offset); + // Check JSON keywords. + let { index: bestKeyIdx, keyword: matchedKeyword } = earliestKeywordIndex(lower, TOOL_SEGMENT_KEYWORDS, offset); + // Also check XML tool tags. + for (const tag of XML_TOOL_SEGMENT_TAGS) { + const idx = lower.indexOf(tag, offset); + if (idx >= 0 && (bestKeyIdx < 0 || idx < bestKeyIdx)) { + bestKeyIdx = idx; + matchedKeyword = tag; + } + } if (bestKeyIdx < 0) { return -1; } + // For XML tags, the '<' is itself the segment start. + if (s[bestKeyIdx] === '<') { + if (!insideCodeFenceWithState(state, s.slice(0, bestKeyIdx))) { + return bestKeyIdx; + } + offset = bestKeyIdx + matchedKeyword.length; + continue; + } const keyIdx = bestKeyIdx; const start = s.slice(0, keyIdx).lastIndexOf('{'); const candidateStart = start >= 0 ? start : keyIdx; @@ -173,6 +204,17 @@ function consumeToolCapture(state, toolNames) { if (!captured) { return { ready: false, prefix: '', calls: [], suffix: '' }; } + + // Try XML tool call extraction first. + const xmlResult = consumeXMLToolCaptureImpl(captured, toolNames, trimWrappingJSONFence); + if (xmlResult.ready) { + return xmlResult; + } + // If XML tags are present but block is incomplete, keep buffering. + if (hasOpenXMLToolTag(captured)) { + return { ready: false, prefix: '', calls: [], suffix: '' }; + } + const lower = captured.toLowerCase(); const { index: keyIdx } = earliestKeywordIndex(lower, TOOL_SEGMENT_KEYWORDS); if (keyIdx < 0) { @@ -231,52 +273,6 @@ function consumeToolCapture(state, toolNames) { }; } -function extractToolHistoryBlock(captured, keyIdx) { - if (typeof captured !== 'string' || keyIdx < 0 || keyIdx >= captured.length) { - return { ok: false, start: 0, end: 0 }; - } - const rest = captured.slice(keyIdx).toLowerCase(); - if (rest.startsWith('[tool_call_history]')) { - const closeTag = '[/tool_call_history]'; - const closeIdx = rest.indexOf(closeTag); - if (closeIdx < 0) { - return { ok: false, start: 0, end: 0 }; - } - return { ok: true, start: keyIdx, end: keyIdx + closeIdx + closeTag.length }; - } - if (rest.startsWith('[tool_result_history]')) { - const closeTag = '[/tool_result_history]'; - const closeIdx = rest.indexOf(closeTag); - if (closeIdx < 0) { - return { ok: false, start: 0, end: 0 }; - } - return { ok: true, start: keyIdx, end: keyIdx + closeIdx + closeTag.length }; - } - return { ok: false, start: 0, end: 0 }; -} - -function trimWrappingJSONFence(prefix, suffix) { - const rightTrimmedPrefix = (prefix || '').replace(/[ \t\r\n]+$/g, ''); - const fenceIdx = rightTrimmedPrefix.lastIndexOf('```'); - if (fenceIdx < 0) return { prefix, suffix }; - const fenceCount = (rightTrimmedPrefix.slice(0, fenceIdx + 3).match(/```/g) || []).length; - if (fenceCount % 2 === 0) { - return { prefix, suffix }; - } - const header = rightTrimmedPrefix.slice(fenceIdx + 3).trim().toLowerCase(); - if (header && header !== 'json') { - return { prefix, suffix }; - } - const leftTrimmedSuffix = (suffix || '').replace(/^[ \t\r\n]+/g, ''); - if (!leftTrimmedSuffix.startsWith('```')) { - return { prefix, suffix }; - } - const consumed = (suffix || '').length - leftTrimmedSuffix.length; - return { - prefix: rightTrimmedPrefix.slice(0, fenceIdx), - suffix: (suffix || '').slice(consumed + 3), - }; -} module.exports = { processToolSieveChunk, flushToolSieve, diff --git a/internal/js/helpers/stream-tool-sieve/tool-keywords.js b/internal/js/helpers/stream-tool-sieve/tool-keywords.js index 76be42e..ea35f1e 100644 --- a/internal/js/helpers/stream-tool-sieve/tool-keywords.js +++ b/internal/js/helpers/stream-tool-sieve/tool-keywords.js @@ -8,6 +8,19 @@ const TOOL_SEGMENT_KEYWORDS = [ '[tool_result_history]', ]; +const XML_TOOL_SEGMENT_TAGS = [ + '', '', '', '', +]; + +const XML_TOOL_OPENING_TAGS = [ + '', '', '', '', '', '', +]; + function earliestKeywordIndex(text, keywords = TOOL_SEGMENT_KEYWORDS, offset = 0) { if (!text) { return { index: -1, keyword: '' }; @@ -26,5 +39,8 @@ function earliestKeywordIndex(text, keywords = TOOL_SEGMENT_KEYWORDS, offset = 0 module.exports = { TOOL_SEGMENT_KEYWORDS, + XML_TOOL_SEGMENT_TAGS, + XML_TOOL_OPENING_TAGS, + XML_TOOL_CLOSING_TAGS, earliestKeywordIndex, }; diff --git a/plans/node-syntax-gate-targets.txt b/plans/node-syntax-gate-targets.txt index 8f97f83..e466ff0 100644 --- a/plans/node-syntax-gate-targets.txt +++ b/plans/node-syntax-gate-targets.txt @@ -16,6 +16,8 @@ internal/js/helpers/stream-tool-sieve.js internal/js/helpers/stream-tool-sieve/index.js internal/js/helpers/stream-tool-sieve/state.js internal/js/helpers/stream-tool-sieve/sieve.js +internal/js/helpers/stream-tool-sieve/sieve-xml.js internal/js/helpers/stream-tool-sieve/jsonscan.js internal/js/helpers/stream-tool-sieve/parse.js internal/js/helpers/stream-tool-sieve/format.js +internal/js/helpers/stream-tool-sieve/tool-keywords.js diff --git a/plans/refactor-line-gate-targets.txt b/plans/refactor-line-gate-targets.txt index c1ffd22..4eed578 100644 --- a/plans/refactor-line-gate-targets.txt +++ b/plans/refactor-line-gate-targets.txt @@ -53,6 +53,7 @@ internal/adapter/openai/responses_stream_runtime_events.go internal/adapter/openai/responses_stream_runtime_toolcalls.go internal/adapter/openai/tool_sieve_state.go internal/adapter/openai/tool_sieve_core.go +internal/adapter/openai/tool_sieve_xml.go internal/adapter/openai/tool_sieve_jsonscan.go internal/util/toolcalls_parse.go @@ -106,6 +107,7 @@ internal/js/helpers/stream-tool-sieve.js internal/js/helpers/stream-tool-sieve/index.js internal/js/helpers/stream-tool-sieve/state.js internal/js/helpers/stream-tool-sieve/sieve.js +internal/js/helpers/stream-tool-sieve/sieve-xml.js internal/js/helpers/stream-tool-sieve/jsonscan.js internal/js/helpers/stream-tool-sieve/parse.js internal/js/helpers/stream-tool-sieve/format.js