From 892213071ad202748e12ead2b47b764d02f14216 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Sun, 8 Mar 2026 00:12:43 +0800 Subject: [PATCH 1/6] Align Go/JS tool-call parsing semantics and compat fixtures --- docs/toolcall-semantics.md | 40 ++++++ internal/compat/go_compat_test.go | 25 ++-- .../js/helpers/stream-tool-sieve/parse.js | 61 +++++++- .../stream-tool-sieve/parse_payload.js | 105 ++++++++++++++ internal/util/toolcalls_markup.go | 134 ++++++++++++++++++ internal/util/toolcalls_parse.go | 44 ++++-- .../expected/toolcalls_allowlist_empty.json | 9 +- .../toolcalls_case_insensitive_canonical.json | 7 +- .../expected/toolcalls_fenced_json.json | 7 +- .../expected/toolcalls_function_call_tag.json | 13 ++ .../expected/toolcalls_invoke_attr.json | 13 ++ .../expected/toolcalls_loose_normalize.json | 13 ++ .../toolcalls_namespace_tail_normalize.json | 13 ++ .../toolcalls_standalone_fenced_example.json | 7 +- .../toolcalls_standalone_mixed_prose.json | 7 +- .../expected/toolcalls_standalone_pure.json | 7 +- .../expected/toolcalls_unknown_name.json | 9 +- .../expected/toolcalls_xml_tool_call.json | 13 ++ .../fixtures/toolcalls/function_call_tag.json | 6 + .../fixtures/toolcalls/invoke_attr.json | 6 + .../fixtures/toolcalls/loose_normalize.json | 6 + .../toolcalls/namespace_tail_normalize.json | 6 + .../fixtures/toolcalls/xml_tool_call.json | 6 + tests/node/js_compat_test.js | 9 +- 24 files changed, 519 insertions(+), 47 deletions(-) create mode 100644 docs/toolcall-semantics.md create mode 100644 internal/util/toolcalls_markup.go create mode 100644 tests/compat/expected/toolcalls_function_call_tag.json create mode 100644 tests/compat/expected/toolcalls_invoke_attr.json create mode 100644 tests/compat/expected/toolcalls_loose_normalize.json create mode 100644 tests/compat/expected/toolcalls_namespace_tail_normalize.json create mode 100644 tests/compat/expected/toolcalls_xml_tool_call.json create mode 100644 tests/compat/fixtures/toolcalls/function_call_tag.json create mode 100644 tests/compat/fixtures/toolcalls/invoke_attr.json create mode 100644 tests/compat/fixtures/toolcalls/loose_normalize.json create mode 100644 tests/compat/fixtures/toolcalls/namespace_tail_normalize.json create mode 100644 tests/compat/fixtures/toolcalls/xml_tool_call.json diff --git a/docs/toolcall-semantics.md b/docs/toolcall-semantics.md new file mode 100644 index 0000000..50165a9 --- /dev/null +++ b/docs/toolcall-semantics.md @@ -0,0 +1,40 @@ +# Tool call parsing semantics (Go canonical spec) + +This document defines the cross-runtime contract for `ParseToolCallsDetailed` / `parseToolCallsDetailed`. + +## Output contract + +- `calls`: accepted tool calls with normalized tool names. +- `sawToolCallSyntax`: true when tool-call-like syntax is detected (`tool_calls`, ``, ``, ``) or a valid call is parsed. +- `rejectedByPolicy`: true when parser extracted call syntax but all calls are rejected by allow-list policy. +- `rejectedToolNames`: de-duplicated rejected tool names in first-seen order. + +## Parse pipeline + +1. Strip fenced code blocks for non-standalone parsing. +2. Build candidates from: + - full text, + - fenced JSON snippets, + - extracted JSON objects around `tool_calls`, + - first `{` to last `}` object slice. +3. Parse each candidate in order: + - JSON payload parser (`tool_calls`, list, single call object), + - markup parser (``, ``, ``; supports attributes + nested fields). +4. Stop at first candidate that yields at least one call. + +## Name normalization policy + +When matching parsed names against configured tools: + +1. exact match, +2. case-insensitive match, +3. namespace tail match (`a.b.c` => `c`), +4. loose alnum match (remove non `[a-z0-9]`, compare). + +## Standalone mode + +Standalone mode (`ParseStandaloneToolCallsDetailed`) parses the whole input directly (no candidate slicing), while still applying: + +- example-context guard, +- JSON then markup fallback, +- the same allow-list normalization policy. diff --git a/internal/compat/go_compat_test.go b/internal/compat/go_compat_test.go index fa68eb2..7768e4b 100644 --- a/internal/compat/go_compat_test.go +++ b/internal/compat/go_compat_test.go @@ -73,22 +73,31 @@ func TestGoCompatToolcallFixtures(t *testing.T) { mustLoadJSON(t, fixturePath, &fixture) var expected struct { - Calls []util.ParsedToolCall `json:"calls"` + Calls []util.ParsedToolCall `json:"calls"` + SawToolCallSyntax bool `json:"sawToolCallSyntax"` + RejectedByPolicy bool `json:"rejectedByPolicy"` + RejectedToolNames []string `json:"rejectedToolNames"` } mustLoadJSON(t, expectedPath, &expected) - var got []util.ParsedToolCall + var got util.ToolCallParseResult switch strings.ToLower(strings.TrimSpace(fixture.Mode)) { case "standalone": - got = util.ParseStandaloneToolCalls(fixture.Text, fixture.ToolNames) + got = util.ParseStandaloneToolCallsDetailed(fixture.Text, fixture.ToolNames) default: - got = util.ParseToolCalls(fixture.Text, fixture.ToolNames) + got = util.ParseToolCallsDetailed(fixture.Text, fixture.ToolNames) } - if len(got) == 0 && len(expected.Calls) == 0 { - continue + if got.Calls == nil { + got.Calls = []util.ParsedToolCall{} } - if !reflect.DeepEqual(got, expected.Calls) { - t.Fatalf("toolcall fixture %s mismatch:\n got=%#v\nwant=%#v", name, got, expected.Calls) + if got.RejectedToolNames == nil { + got.RejectedToolNames = []string{} + } + if !reflect.DeepEqual(got.Calls, expected.Calls) || + got.SawToolCallSyntax != expected.SawToolCallSyntax || + got.RejectedByPolicy != expected.RejectedByPolicy || + !reflect.DeepEqual(got.RejectedToolNames, expected.RejectedToolNames) { + t.Fatalf("toolcall fixture %s mismatch:\n got=%#v\nwant=%#v", name, got, expected) } } } diff --git a/internal/js/helpers/stream-tool-sieve/parse.js b/internal/js/helpers/stream-tool-sieve/parse.js index bf21046..1b882ff 100644 --- a/internal/js/helpers/stream-tool-sieve/parse.js +++ b/internal/js/helpers/stream-tool-sieve/parse.js @@ -8,8 +8,11 @@ const { stripFencedCodeBlocks, buildToolCallCandidates, parseToolCallsPayload, + parseMarkupToolCalls, } = require('./parse_payload'); +const TOOL_NAME_LOOSE_PATTERN = /[^a-z0-9]+/g; + function extractToolNames(tools) { if (!Array.isArray(tools) || tools.length === 0) { return []; @@ -41,12 +44,15 @@ function parseToolCallsDetailed(text, toolNames) { if (!toStringSafe(sanitized)) { return result; } - result.sawToolCallSyntax = sanitized.toLowerCase().includes('tool_calls'); + result.sawToolCallSyntax = looksLikeToolCallSyntax(sanitized); const candidates = buildToolCallCandidates(sanitized); let parsed = []; for (const c of candidates) { parsed = parseToolCallsPayload(c); + if (parsed.length === 0) { + parsed = parseMarkupToolCalls(c); + } if (parsed.length > 0) { result.sawToolCallSyntax = true; break; @@ -73,15 +79,17 @@ function parseStandaloneToolCallsDetailed(text, toolNames) { if (!trimmed) { return result; } + if (trimmed.includes('```')) { + return result; + } if (looksLikeToolExampleContext(trimmed)) { return result; } - result.sawToolCallSyntax = trimmed.toLowerCase().includes('tool_calls'); - if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { - return result; + result.sawToolCallSyntax = looksLikeToolCallSyntax(trimmed); + let parsed = parseToolCallsPayload(trimmed); + if (parsed.length === 0) { + parsed = parseMarkupToolCalls(trimmed); } - - const parsed = parseToolCallsPayload(trimmed); if (parsed.length === 0) { return result; } @@ -146,7 +154,7 @@ function filterToolCallsDetailed(parsed, toolNames) { if (allowed.has(tc.name)) { matchedName = tc.name; } else { - matchedName = allowedCanonical.get(tc.name.toLowerCase()) || ''; + matchedName = resolveAllowedToolName(tc.name, allowed, allowedCanonical); } if (!matchedName) { if (!seenRejected.has(tc.name)) { @@ -163,6 +171,45 @@ function filterToolCallsDetailed(parsed, toolNames) { return { calls, rejectedToolNames: rejected }; } +function resolveAllowedToolName(name, allowed, allowedCanonical) { + const normalizedName = toStringSafe(name).trim(); + if (!normalizedName) { + return ''; + } + if (allowed.has(normalizedName)) { + return normalizedName; + } + const lower = normalizedName.toLowerCase(); + if (allowedCanonical.has(lower)) { + return allowedCanonical.get(lower); + } + const idx = lower.lastIndexOf('.'); + if (idx >= 0 && idx < lower.length - 1) { + const tail = lower.slice(idx + 1); + if (allowedCanonical.has(tail)) { + return allowedCanonical.get(tail); + } + } + const loose = lower.replace(TOOL_NAME_LOOSE_PATTERN, ''); + if (!loose) { + return ''; + } + for (const [candidateLower, canonical] of allowedCanonical.entries()) { + if (candidateLower.replace(TOOL_NAME_LOOSE_PATTERN, '') === loose) { + return canonical; + } + } + return ''; +} + +function looksLikeToolCallSyntax(text) { + const lower = toStringSafe(text).toLowerCase(); + return lower.includes('tool_calls') + || lower.includes(']*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi; +const TOOL_CALL_MARKUP_SELFCLOSE_PATTERN = /<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)\/>/gi; +const TOOL_CALL_MARKUP_NAME_TAG_PATTERN = /<(?:[a-z0-9_:-]+:)?(name|function)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/i; +const TOOL_CALL_MARKUP_ARGS_TAG_PATTERN = /<(?:[a-z0-9_:-]+:)?(input|arguments|argument|parameters|parameter|args|params)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/i; +const TOOL_CALL_MARKUP_KV_PATTERN = /<(?:[a-z0-9_:-]+:)?([a-z0-9_.-]+)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi; +const TOOL_CALL_MARKUP_ATTR_PATTERN = /(name|function|tool)\s*=\s*"([^"]+)"/i; const { toStringSafe, @@ -103,6 +109,104 @@ function parseToolCallsPayload(payload) { return one ? [one] : []; } +function parseMarkupToolCalls(text) { + const raw = toStringSafe(text).trim(); + if (!raw) { + return []; + } + const out = []; + for (const m of raw.matchAll(TOOL_CALL_MARKUP_BLOCK_PATTERN)) { + const parsed = parseMarkupSingleToolCall(toStringSafe(m[2]).trim(), toStringSafe(m[3]).trim()); + if (parsed) { + out.push(parsed); + } + } + for (const m of raw.matchAll(TOOL_CALL_MARKUP_SELFCLOSE_PATTERN)) { + const parsed = parseMarkupSingleToolCall(toStringSafe(m[1]).trim(), ''); + if (parsed) { + out.push(parsed); + } + } + return out; +} + +function parseMarkupSingleToolCall(attrs, inner) { + const embedded = parseToolCallsPayload(inner); + if (embedded.length > 0) { + return embedded[0]; + } + let name = ''; + const attrMatch = attrs.match(TOOL_CALL_MARKUP_ATTR_PATTERN); + if (attrMatch && attrMatch[2]) { + name = toStringSafe(attrMatch[2]).trim(); + } + if (!name) { + const m = inner.match(TOOL_CALL_MARKUP_NAME_TAG_PATTERN); + if (m && m[2]) { + name = stripTagText(m[2]); + } + } + if (!name) { + return null; + } + + let input = {}; + const argsMatch = inner.match(TOOL_CALL_MARKUP_ARGS_TAG_PATTERN); + if (argsMatch && argsMatch[2]) { + input = parseMarkupInput(argsMatch[2]); + } else { + const kv = parseMarkupKVObject(inner); + if (Object.keys(kv).length > 0) { + input = kv; + } + } + return { name, input }; +} + +function parseMarkupInput(raw) { + const s = toStringSafe(raw).trim(); + if (!s) { + return {}; + } + const parsed = parseToolCallInput(s); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && Object.keys(parsed).length > 0) { + return parsed; + } + const kv = parseMarkupKVObject(s); + if (Object.keys(kv).length > 0) { + return kv; + } + return { _raw: stripTagText(s) }; +} + +function parseMarkupKVObject(text) { + const raw = toStringSafe(text).trim(); + if (!raw) { + return {}; + } + const out = {}; + for (const m of raw.matchAll(TOOL_CALL_MARKUP_KV_PATTERN)) { + const key = toStringSafe(m[1]).trim(); + if (!key) { + continue; + } + const valueRaw = stripTagText(m[2]); + if (!valueRaw) { + continue; + } + try { + out[key] = JSON.parse(valueRaw); + } catch (_err) { + out[key] = valueRaw; + } + } + return out; +} + +function stripTagText(text) { + return toStringSafe(text).replace(/<[^>]+>/g, ' ').trim(); +} + function parseToolCallList(v) { if (!Array.isArray(v)) { return []; @@ -193,4 +297,5 @@ module.exports = { stripFencedCodeBlocks, buildToolCallCandidates, parseToolCallsPayload, + parseMarkupToolCalls, }; diff --git a/internal/util/toolcalls_markup.go b/internal/util/toolcalls_markup.go new file mode 100644 index 0000000..1fef7a7 --- /dev/null +++ b/internal/util/toolcalls_markup.go @@ -0,0 +1,134 @@ +package util + +import ( + "encoding/json" + "regexp" + "strings" +) + +var toolCallMarkupTagNames = []string{"tool_call", "function_call", "invoke"} +var toolCallMarkupTagPatternByName = map[string]*regexp.Regexp{ + "tool_call": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?tool_call\b([^>]*)>(.*?)`), + "function_call": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function_call\b([^>]*)>(.*?)`), + "invoke": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)>(.*?)`), +} +var toolCallMarkupSelfClosingPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)/>`) +var toolCallMarkupNameTagPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?(?:name|function)\b[^>]*>(.*?)`) +var toolCallMarkupArgsTagPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?(?:input|arguments|argument|parameters|parameter|args|params)\b[^>]*>(.*?)`) +var toolCallMarkupKVPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)\b[^>]*>(.*?)`) +var toolCallMarkupAttrPattern = regexp.MustCompile(`(?is)(name|function|tool)\s*=\s*"([^"]+)"`) +var anyTagPattern = regexp.MustCompile(`(?is)<[^>]+>`) + +func parseMarkupToolCalls(text string) []ParsedToolCall { + trimmed := strings.TrimSpace(text) + if trimmed == "" { + return nil + } + + out := make([]ParsedToolCall, 0) + for _, tagName := range toolCallMarkupTagNames { + pattern := toolCallMarkupTagPatternByName[tagName] + for _, m := range pattern.FindAllStringSubmatch(trimmed, -1) { + if len(m) < 3 { + continue + } + attrs := strings.TrimSpace(m[1]) + inner := strings.TrimSpace(m[2]) + if parsed := parseMarkupSingleToolCall(attrs, inner); parsed.Name != "" { + out = append(out, parsed) + } + } + } + for _, m := range toolCallMarkupSelfClosingPattern.FindAllStringSubmatch(trimmed, -1) { + if len(m) < 2 { + continue + } + if parsed := parseMarkupSingleToolCall(strings.TrimSpace(m[1]), ""); parsed.Name != "" { + out = append(out, parsed) + } + } + if len(out) == 0 { + return nil + } + return out +} + +func parseMarkupSingleToolCall(attrs string, inner string) ParsedToolCall { + if parsed := parseToolCallsPayload(inner); len(parsed) > 0 { + return parsed[0] + } + + name := "" + if m := toolCallMarkupAttrPattern.FindStringSubmatch(attrs); len(m) >= 3 { + name = strings.TrimSpace(m[2]) + } + if name == "" { + if m := toolCallMarkupNameTagPattern.FindStringSubmatch(inner); len(m) >= 2 { + name = strings.TrimSpace(stripTagText(m[1])) + } + } + if name == "" { + return ParsedToolCall{} + } + + input := map[string]any{} + if m := toolCallMarkupArgsTagPattern.FindStringSubmatch(inner); len(m) >= 2 { + input = parseMarkupInput(m[1]) + } else if kv := parseMarkupKVObject(inner); len(kv) > 0 { + input = kv + } + return ParsedToolCall{Name: name, Input: input} +} + +func parseMarkupInput(raw string) map[string]any { + raw = strings.TrimSpace(raw) + if raw == "" { + return map[string]any{} + } + if parsed := parseToolCallInput(raw); len(parsed) > 0 { + return parsed + } + if kv := parseMarkupKVObject(raw); len(kv) > 0 { + return kv + } + return map[string]any{"_raw": stripTagText(raw)} +} + +func parseMarkupKVObject(text string) map[string]any { + matches := toolCallMarkupKVPattern.FindAllStringSubmatch(strings.TrimSpace(text), -1) + if len(matches) == 0 { + return nil + } + out := map[string]any{} + for _, m := range matches { + if len(m) < 4 { + continue + } + key := strings.TrimSpace(m[1]) + endKey := strings.TrimSpace(m[3]) + if key == "" { + continue + } + if !strings.EqualFold(key, endKey) { + continue + } + value := strings.TrimSpace(stripTagText(m[2])) + if value == "" { + continue + } + var jsonValue any + if json.Unmarshal([]byte(value), &jsonValue) == nil { + out[key] = jsonValue + continue + } + out[key] = value + } + if len(out) == 0 { + return nil + } + return out +} + +func stripTagText(text string) string { + return strings.TrimSpace(anyTagPattern.ReplaceAllString(text, "")) +} diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 6e949b1..c3e76bf 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -33,12 +33,16 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa if strings.TrimSpace(text) == "" { return result } - result.SawToolCallSyntax = strings.Contains(strings.ToLower(text), "tool_calls") + result.SawToolCallSyntax = looksLikeToolCallSyntax(text) candidates := buildToolCallCandidates(text) var parsed []ParsedToolCall for _, candidate := range candidates { - if tc := parseToolCallsPayload(candidate); len(tc) > 0 { + tc := parseToolCallsPayload(candidate) + if len(tc) == 0 { + tc = parseMarkupToolCalls(candidate) + } + if len(tc) > 0 { parsed = tc result.SawToolCallSyntax = true break @@ -68,17 +72,18 @@ func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) if looksLikeToolExampleContext(trimmed) { return result } - result.SawToolCallSyntax = strings.Contains(strings.ToLower(trimmed), "tool_calls") + result.SawToolCallSyntax = looksLikeToolCallSyntax(trimmed) candidates := []string{trimmed} for _, candidate := range candidates { candidate = strings.TrimSpace(candidate) if candidate == "" { continue } - if !strings.HasPrefix(candidate, "{") && !strings.HasPrefix(candidate, "[") { - continue + parsed := parseToolCallsPayload(candidate) + if len(parsed) == 0 { + parsed = parseMarkupToolCalls(candidate) } - if parsed := parseToolCallsPayload(candidate); len(parsed) > 0 { + if len(parsed) > 0 { result.SawToolCallSyntax = true calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) result.Calls = calls @@ -106,27 +111,32 @@ func filterToolCallsDetailed(parsed []ParsedToolCall, availableToolNames []strin } if len(allowed) == 0 { rejectedSet := map[string]struct{}{} + rejected := make([]string, 0, len(parsed)) for _, tc := range parsed { if tc.Name == "" { continue } + if _, ok := rejectedSet[tc.Name]; ok { + continue + } rejectedSet[tc.Name] = struct{}{} - } - rejected := make([]string, 0, len(rejectedSet)) - for name := range rejectedSet { - rejected = append(rejected, name) + rejected = append(rejected, tc.Name) } return nil, rejected } out := make([]ParsedToolCall, 0, len(parsed)) rejectedSet := map[string]struct{}{} + rejected := make([]string, 0) for _, tc := range parsed { if tc.Name == "" { continue } matchedName := resolveAllowedToolName(tc.Name, allowed, allowedCanonical) if matchedName == "" { - rejectedSet[tc.Name] = struct{}{} + if _, ok := rejectedSet[tc.Name]; !ok { + rejectedSet[tc.Name] = struct{}{} + rejected = append(rejected, tc.Name) + } continue } tc.Name = matchedName @@ -135,10 +145,6 @@ func filterToolCallsDetailed(parsed []ParsedToolCall, availableToolNames []strin } out = append(out, tc) } - rejected := make([]string, 0, len(rejectedSet)) - for name := range rejectedSet { - rejected = append(rejected, name) - } return out, rejected } @@ -186,6 +192,14 @@ func parseToolCallsPayload(payload string) []ParsedToolCall { return nil } +func looksLikeToolCallSyntax(text string) bool { + lower := strings.ToLower(text) + return strings.Contains(lower, "tool_calls") || + strings.Contains(lower, "read_file{\"path\":\"README.MD\"}", + "tool_names": [ + "read_file" + ] +} \ No newline at end of file diff --git a/tests/compat/fixtures/toolcalls/invoke_attr.json b/tests/compat/fixtures/toolcalls/invoke_attr.json new file mode 100644 index 0000000..70c77fc --- /dev/null +++ b/tests/compat/fixtures/toolcalls/invoke_attr.json @@ -0,0 +1,6 @@ +{ + "text": "{\"path\":\"README.MD\"}", + "tool_names": [ + "read_file" + ] +} \ No newline at end of file diff --git a/tests/compat/fixtures/toolcalls/loose_normalize.json b/tests/compat/fixtures/toolcalls/loose_normalize.json new file mode 100644 index 0000000..f4d112e --- /dev/null +++ b/tests/compat/fixtures/toolcalls/loose_normalize.json @@ -0,0 +1,6 @@ +{ + "text": "{\"tool_calls\":[{\"name\":\"read-file\",\"input\":{\"path\":\"README.MD\"}}]}", + "tool_names": [ + "read_file" + ] +} \ No newline at end of file diff --git a/tests/compat/fixtures/toolcalls/namespace_tail_normalize.json b/tests/compat/fixtures/toolcalls/namespace_tail_normalize.json new file mode 100644 index 0000000..67d504d --- /dev/null +++ b/tests/compat/fixtures/toolcalls/namespace_tail_normalize.json @@ -0,0 +1,6 @@ +{ + "text": "{\"tool_calls\":[{\"name\":\"company.fs.read_file\",\"input\":{\"path\":\"README.MD\"}}]}", + "tool_names": [ + "read_file" + ] +} \ No newline at end of file diff --git a/tests/compat/fixtures/toolcalls/xml_tool_call.json b/tests/compat/fixtures/toolcalls/xml_tool_call.json new file mode 100644 index 0000000..279f1a2 --- /dev/null +++ b/tests/compat/fixtures/toolcalls/xml_tool_call.json @@ -0,0 +1,6 @@ +{ + "text": "read_file{\"path\":\"README.MD\"}", + "tool_names": [ + "read_file" + ] +} \ No newline at end of file diff --git a/tests/node/js_compat_test.js b/tests/node/js_compat_test.js index 74b3fd1..c4bdcff 100644 --- a/tests/node/js_compat_test.js +++ b/tests/node/js_compat_test.js @@ -6,7 +6,7 @@ const fs = require('node:fs'); const path = require('node:path'); const chatStream = require('../../api/chat-stream.js'); -const { parseToolCalls, parseStandaloneToolCalls } = require('../../internal/js/helpers/stream-tool-sieve.js'); +const { parseToolCallsDetailed, parseStandaloneToolCallsDetailed } = require('../../internal/js/helpers/stream-tool-sieve.js'); const { parseChunkForContent, estimateTokens } = chatStream.__test; @@ -44,9 +44,12 @@ test('js compat: toolcall fixtures', () => { const fixture = readJSON(path.join(fixtureDir, file)); const expected = readJSON(path.join(expectedDir, `toolcalls_${name}.json`)); const mode = typeof fixture.mode === 'string' ? fixture.mode.trim().toLowerCase() : ''; - const parser = mode === 'standalone' ? parseStandaloneToolCalls : parseToolCalls; + const parser = mode === 'standalone' ? parseStandaloneToolCallsDetailed : parseToolCallsDetailed; const got = parser(fixture.text, fixture.tool_names || []); - assert.deepEqual(got, expected.calls, `${name}: calls mismatch`); + assert.deepEqual(got.calls, expected.calls, `${name}: calls mismatch`); + assert.equal(got.sawToolCallSyntax, expected.sawToolCallSyntax, `${name}: sawToolCallSyntax mismatch`); + assert.equal(got.rejectedByPolicy, expected.rejectedByPolicy, `${name}: rejectedByPolicy mismatch`); + assert.deepEqual(got.rejectedToolNames, expected.rejectedToolNames, `${name}: rejectedToolNames mismatch`); } }); From 9b93badb57b7f66bf382bdc593df2417b096649d Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Sun, 8 Mar 2026 00:55:32 +0800 Subject: [PATCH 2/6] Harden markup tag parsing to avoid mismatched-tag false positives --- .../stream-tool-sieve/parse_payload.js | 37 +++++++++++++---- internal/util/toolcalls_markup.go | 41 +++++++++++++++---- internal/util/toolcalls_test.go | 8 ++++ tests/node/stream-tool-sieve.test.js | 6 +++ 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/internal/js/helpers/stream-tool-sieve/parse_payload.js b/internal/js/helpers/stream-tool-sieve/parse_payload.js index 8779f59..612e186 100644 --- a/internal/js/helpers/stream-tool-sieve/parse_payload.js +++ b/internal/js/helpers/stream-tool-sieve/parse_payload.js @@ -3,10 +3,21 @@ const TOOL_CALL_PATTERN = /\{\s*["']tool_calls["']\s*:\s*\[(.*?)\]\s*\}/s; const TOOL_CALL_MARKUP_BLOCK_PATTERN = /<(?:[a-z0-9_:-]+:)?(tool_call|function_call|invoke)\b([^>]*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi; const TOOL_CALL_MARKUP_SELFCLOSE_PATTERN = /<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)\/>/gi; -const TOOL_CALL_MARKUP_NAME_TAG_PATTERN = /<(?:[a-z0-9_:-]+:)?(name|function)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/i; -const TOOL_CALL_MARKUP_ARGS_TAG_PATTERN = /<(?:[a-z0-9_:-]+:)?(input|arguments|argument|parameters|parameter|args|params)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/i; const TOOL_CALL_MARKUP_KV_PATTERN = /<(?:[a-z0-9_:-]+:)?([a-z0-9_.-]+)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi; const TOOL_CALL_MARKUP_ATTR_PATTERN = /(name|function|tool)\s*=\s*"([^"]+)"/i; +const TOOL_CALL_MARKUP_NAME_PATTERNS = [ + /<(?:[a-z0-9_:-]+:)?name\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?name>/i, + /<(?:[a-z0-9_:-]+:)?function\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?function>/i, +]; +const TOOL_CALL_MARKUP_ARGS_PATTERNS = [ + /<(?:[a-z0-9_:-]+:)?input\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?input>/i, + /<(?:[a-z0-9_:-]+:)?arguments\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?arguments>/i, + /<(?:[a-z0-9_:-]+:)?argument\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?argument>/i, + /<(?:[a-z0-9_:-]+:)?parameters\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?parameters>/i, + /<(?:[a-z0-9_:-]+:)?parameter\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?parameter>/i, + /<(?:[a-z0-9_:-]+:)?args\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?args>/i, + /<(?:[a-z0-9_:-]+:)?params\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?params>/i, +]; const { toStringSafe, @@ -141,19 +152,16 @@ function parseMarkupSingleToolCall(attrs, inner) { name = toStringSafe(attrMatch[2]).trim(); } if (!name) { - const m = inner.match(TOOL_CALL_MARKUP_NAME_TAG_PATTERN); - if (m && m[2]) { - name = stripTagText(m[2]); - } + name = stripTagText(findMarkupTagValue(inner, TOOL_CALL_MARKUP_NAME_PATTERNS)); } if (!name) { return null; } let input = {}; - const argsMatch = inner.match(TOOL_CALL_MARKUP_ARGS_TAG_PATTERN); - if (argsMatch && argsMatch[2]) { - input = parseMarkupInput(argsMatch[2]); + const argsRaw = findMarkupTagValue(inner, TOOL_CALL_MARKUP_ARGS_PATTERNS); + if (argsRaw) { + input = parseMarkupInput(argsRaw); } else { const kv = parseMarkupKVObject(inner); if (Object.keys(kv).length > 0) { @@ -207,6 +215,17 @@ function stripTagText(text) { return toStringSafe(text).replace(/<[^>]+>/g, ' ').trim(); } +function findMarkupTagValue(text, patterns) { + const source = toStringSafe(text); + for (const p of patterns) { + const m = source.match(p); + if (m && m[1]) { + return toStringSafe(m[1]); + } + } + return ''; +} + function parseToolCallList(v) { if (!Array.isArray(v)) { return []; diff --git a/internal/util/toolcalls_markup.go b/internal/util/toolcalls_markup.go index 1fef7a7..cc0f8bb 100644 --- a/internal/util/toolcalls_markup.go +++ b/internal/util/toolcalls_markup.go @@ -13,11 +13,24 @@ var toolCallMarkupTagPatternByName = map[string]*regexp.Regexp{ "invoke": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)>(.*?)`), } var toolCallMarkupSelfClosingPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)/>`) -var toolCallMarkupNameTagPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?(?:name|function)\b[^>]*>(.*?)`) -var toolCallMarkupArgsTagPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?(?:input|arguments|argument|parameters|parameter|args|params)\b[^>]*>(.*?)`) var toolCallMarkupKVPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)\b[^>]*>(.*?)`) var toolCallMarkupAttrPattern = regexp.MustCompile(`(?is)(name|function|tool)\s*=\s*"([^"]+)"`) var anyTagPattern = regexp.MustCompile(`(?is)<[^>]+>`) +var toolCallMarkupNameTagNames = []string{"name", "function"} +var toolCallMarkupNamePatternByTag = map[string]*regexp.Regexp{ + "name": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?name\b[^>]*>(.*?)`), + "function": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function\b[^>]*>(.*?)`), +} +var toolCallMarkupArgsTagNames = []string{"input", "arguments", "argument", "parameters", "parameter", "args", "params"} +var toolCallMarkupArgsPatternByTag = map[string]*regexp.Regexp{ + "input": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?input\b[^>]*>(.*?)`), + "arguments": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?arguments\b[^>]*>(.*?)`), + "argument": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?argument\b[^>]*>(.*?)`), + "parameters": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?parameters\b[^>]*>(.*?)`), + "parameter": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?parameter\b[^>]*>(.*?)`), + "args": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?args\b[^>]*>(.*?)`), + "params": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?params\b[^>]*>(.*?)`), +} func parseMarkupToolCalls(text string) []ParsedToolCall { trimmed := strings.TrimSpace(text) @@ -63,17 +76,15 @@ func parseMarkupSingleToolCall(attrs string, inner string) ParsedToolCall { name = strings.TrimSpace(m[2]) } if name == "" { - if m := toolCallMarkupNameTagPattern.FindStringSubmatch(inner); len(m) >= 2 { - name = strings.TrimSpace(stripTagText(m[1])) - } + name = findMarkupTagValue(inner, toolCallMarkupNameTagNames, toolCallMarkupNamePatternByTag) } if name == "" { return ParsedToolCall{} } input := map[string]any{} - if m := toolCallMarkupArgsTagPattern.FindStringSubmatch(inner); len(m) >= 2 { - input = parseMarkupInput(m[1]) + if argsRaw := findMarkupTagValue(inner, toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); argsRaw != "" { + input = parseMarkupInput(argsRaw) } else if kv := parseMarkupKVObject(inner); len(kv) > 0 { input = kv } @@ -132,3 +143,19 @@ func parseMarkupKVObject(text string) map[string]any { func stripTagText(text string) string { return strings.TrimSpace(anyTagPattern.ReplaceAllString(text, "")) } + +func findMarkupTagValue(text string, tagNames []string, patternByTag map[string]*regexp.Regexp) string { + for _, tag := range tagNames { + pattern := patternByTag[tag] + if pattern == nil { + continue + } + if m := pattern.FindStringSubmatch(text); len(m) >= 2 { + value := strings.TrimSpace(m[1]) + if value != "" { + return value + } + } + } + return "" +} diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index 0e682dc..e830092 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -137,3 +137,11 @@ func TestParseToolCallsAllowsPunctuationVariantToolName(t *testing.T) { t.Fatalf("expected canonical tool name read_file, got %q", calls[0].Name) } } + +func TestParseToolCallsDoesNotAcceptMismatchedMarkupTags(t *testing.T) { + text := `read_file{"path":"README.md"}` + calls := ParseToolCalls(text, []string{"read_file"}) + if len(calls) != 0 { + t.Fatalf("expected mismatched tags to be rejected, got %#v", calls) + } +} diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index e68f2ff..20c00b8 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -249,3 +249,9 @@ test('formatOpenAIStreamToolCalls reuses ids with the same idStore', () => { assert.equal(second.length, 1); assert.equal(first[0].id, second[0].id); }); + +test('parseToolCalls rejects mismatched markup tags', () => { + const payload = 'read_file{"path":"README.md"}'; + const calls = parseToolCalls(payload, ['read_file']); + assert.equal(calls.length, 0); +}); From 60e9d707d4ad97149f3b5f810c55ff1e642378a2 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Sun, 8 Mar 2026 01:10:53 +0800 Subject: [PATCH 3/6] Merge origin/dev into PR branch and resolve toolcall test conflicts --- README.MD | 8 + README.en.md | 8 + .../adapter/claude/handler_stream_test.go | 75 ++++++ internal/adapter/claude/handler_util_test.go | 7 +- internal/adapter/claude/handler_utils.go | 2 +- .../adapter/claude/stream_runtime_core.go | 17 ++ internal/adapter/openai/message_normalize.go | 2 +- .../adapter/openai/message_normalize_test.go | 23 ++ .../responses_stream_runtime_toolcalls.go | 10 + .../adapter/openai/responses_stream_test.go | 34 +++ .../format/openai/render_stream_events.go | 13 ++ .../js/helpers/stream-tool-sieve/sieve.js | 20 +- internal/util/toolcalls_parse.go | 12 +- internal/util/toolcalls_parse_markup.go | 219 ++++++++++++++++++ internal/util/toolcalls_test.go | 134 +++++++++++ tests/node/stream-tool-sieve.test.js | 27 ++- 16 files changed, 586 insertions(+), 25 deletions(-) create mode 100644 internal/util/toolcalls_parse_markup.go diff --git a/README.MD b/README.MD index 0ce3ec7..d3dc05d 100644 --- a/README.MD +++ b/README.MD @@ -106,6 +106,14 @@ flowchart LR 可通过配置中的 `claude_mapping` 或 `claude_model_mapping` 覆盖映射关系。 另外，`/anthropic/v1/models` 现已包含 Claude 1.x/2.x/3.x/4.x 历史模型 ID 与常见别名，便于旧客户端直接兼容。 + +#### Claude Code 接入避坑（实测） + +- `ANTHROPIC_BASE_URL` 推荐直接指向 DS2API 根地址（例如 `http://127.0.0.1:5001`），Claude Code 会请求 `/v1/messages?beta=true`。 +- `ANTHROPIC_API_KEY` 需要与 `config.json` 中 `keys` 一致；建议同时保留常规 key 与 `sk-ant-*` 形态 key，兼容不同客户端校验习惯。 +- 若系统设置了代理，建议对 DS2API 地址配置 `NO_PROXY=127.0.0.1,localhost,<你的主机IP>`，避免本地回环请求被代理拦截。 +- 如遇“工具调用输出成文本、未执行”问题，请升级到包含 Claude 工具调用多格式解析（JSON/XML/ANTML/invoke）的版本。 + ### Gemini 接口 Gemini 适配器将模型名通过 `model_aliases` 或内置规则映射到 DeepSeek 原生模型，支持 `generateContent` 和 `streamGenerateContent` 两种调用方式，并完整支持 Tool Calling（`functionDeclarations` → `functionCall` 输出）。 diff --git a/README.en.md b/README.en.md index 72d8bd8..1c07c23 100644 --- a/README.en.md +++ b/README.en.md @@ -106,6 +106,14 @@ flowchart LR Override mapping via `claude_mapping` or `claude_model_mapping` in config. In addition, `/anthropic/v1/models` now includes historical Claude 1.x/2.x/3.x/4.x IDs and common aliases for legacy client compatibility. + +#### Claude Code integration pitfalls (validated) + +- Set `ANTHROPIC_BASE_URL` to the DS2API root URL (for example `http://127.0.0.1:5001`). Claude Code sends requests to `/v1/messages?beta=true`. +- `ANTHROPIC_API_KEY` must match an entry in `keys` from `config.json`. Keeping both a regular key and an `sk-ant-*` style key improves client compatibility. +- If your environment has proxy variables, set `NO_PROXY=127.0.0.1,localhost,` for DS2API to avoid proxy interception of local traffic. +- If tool calls are rendered as plain text and not executed, upgrade to a build that includes multi-format Claude tool-call parsing (JSON/XML/ANTML/invoke). + ### Gemini Endpoint The Gemini adapter maps model names to DeepSeek native models via `model_aliases` or built-in heuristics, supporting both `generateContent` and `streamGenerateContent` call patterns with full Tool Calling support (`functionDeclarations` → `functionCall` output). diff --git a/internal/adapter/claude/handler_stream_test.go b/internal/adapter/claude/handler_stream_test.go index ebce879..dda425a 100644 --- a/internal/adapter/claude/handler_stream_test.go +++ b/internal/adapter/claude/handler_stream_test.go @@ -315,3 +315,78 @@ func asString(v any) string { s, _ := v.(string) return s } + +func TestHandleClaudeStreamRealtimeToolSafetyAcrossStructuredFormats(t *testing.T) { + tests := []struct { + name string + payload string + }{ + {name: "xml_tool_call", payload: `Bashpwd`}, + {name: "xml_json_tool_call", payload: `{"tool":"Bash","params":{"command":"pwd"}}`}, + {name: "nested_tool_tag_style", payload: `pwd`}, + {name: "function_tag_style", payload: `Bashpwd`}, + {name: "antml_argument_style", payload: `pwd`}, + {name: "antml_function_attr_parameters", payload: `{"command":"pwd"}`}, + {name: "invoke_parameter_style", payload: `pwd`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := &Handler{} + resp := makeClaudeSSEHTTPResponse( + `data: {"p":"response/content","v":"`+strings.ReplaceAll(tc.payload, `"`, `\"`)+`"}`, + `data: [DONE]`, + ) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/anthropic/v1/messages", nil) + + h.handleClaudeStreamRealtime(rec, req, resp, "claude-sonnet-4-5", []any{map[string]any{"role": "user", "content": "use tool"}}, false, false, []string{"Bash"}) + + frames := parseClaudeFrames(t, rec.Body.String()) + foundToolUse := false + for _, f := range findClaudeFrames(frames, "content_block_start") { + contentBlock, _ := f.Payload["content_block"].(map[string]any) + if contentBlock["type"] == "tool_use" { + foundToolUse = true + break + } + } + if !foundToolUse { + t.Fatalf("expected tool_use block for format %s, body=%s", tc.name, rec.Body.String()) + } + }) + } +} + +func TestHandleClaudeStreamRealtimeDoesNotStopOnUnclosedFencedToolExample(t *testing.T) { + h := &Handler{} + resp := makeClaudeSSEHTTPResponse( + "data: {\"p\":\"response/content\",\"v\":\"Here is an example:\\n```json\\n{\\\"tool_calls\\\":[{\\\"name\\\":\\\"Bash\\\",\\\"input\\\":{\\\"command\\\":\\\"pwd\\\"}}]}\"}", + "data: {\"p\":\"response/content\",\"v\":\"\\n```\\nDo not execute it.\"}", + `data: [DONE]`, + ) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/anthropic/v1/messages", nil) + + h.handleClaudeStreamRealtime(rec, req, resp, "claude-sonnet-4-5", []any{map[string]any{"role": "user", "content": "show example only"}}, false, false, []string{"Bash"}) + + frames := parseClaudeFrames(t, rec.Body.String()) + for _, f := range findClaudeFrames(frames, "content_block_start") { + contentBlock, _ := f.Payload["content_block"].(map[string]any) + if contentBlock["type"] == "tool_use" { + t.Fatalf("unexpected tool_use for fenced example, body=%s", rec.Body.String()) + } + } + + foundEndTurn := false + for _, f := range findClaudeFrames(frames, "message_delta") { + delta, _ := f.Payload["delta"].(map[string]any) + if delta["stop_reason"] == "end_turn" { + foundEndTurn = true + break + } + } + if !foundEndTurn { + t.Fatalf("expected stop_reason=end_turn, body=%s", rec.Body.String()) + } +} diff --git a/internal/adapter/claude/handler_util_test.go b/internal/adapter/claude/handler_util_test.go index f5b0ad5..b6c009a 100644 --- a/internal/adapter/claude/handler_util_test.go +++ b/internal/adapter/claude/handler_util_test.go @@ -125,8 +125,11 @@ func TestBuildClaudeToolPromptSingleTool(t *testing.T) { if !containsStr(prompt, "Search the web") { t.Fatalf("expected description in prompt") } - if !containsStr(prompt, "tool_calls") { - t.Fatalf("expected tool_calls instruction in prompt") + if !containsStr(prompt, "tool_use") { + t.Fatalf("expected tool_use instruction in prompt") + } + if containsStr(prompt, "tool_calls") { + t.Fatalf("expected prompt to avoid tool_calls JSON instruction") } } diff --git a/internal/adapter/claude/handler_utils.go b/internal/adapter/claude/handler_utils.go index 3728ffb..2f0c08a 100644 --- a/internal/adapter/claude/handler_utils.go +++ b/internal/adapter/claude/handler_utils.go @@ -51,7 +51,7 @@ func buildClaudeToolPrompt(tools []any) string { parts = append(parts, fmt.Sprintf("Tool: %s\nDescription: %s\nParameters: %s", name, desc, schema)) } parts = append(parts, - "When you need to use tools, you can call multiple tools in one response. Output ONLY JSON like {\"tool_calls\":[{\"name\":\"tool\",\"input\":{}}]}", + "When you need a tool, respond with Claude-native tool use (tool_use) using the provided tool schema. Do not print tool-call JSON in text.", "History markers in conversation: [TOOL_CALL_HISTORY]...[/TOOL_CALL_HISTORY] are your previous tool calls; [TOOL_RESULT_HISTORY]...[/TOOL_RESULT_HISTORY] are runtime tool outputs, not user input.", "After a valid [TOOL_RESULT_HISTORY], continue with final answer instead of repeating the same call unless required fields are still missing.", ) diff --git a/internal/adapter/claude/stream_runtime_core.go b/internal/adapter/claude/stream_runtime_core.go index cb24bdd..fead90a 100644 --- a/internal/adapter/claude/stream_runtime_core.go +++ b/internal/adapter/claude/stream_runtime_core.go @@ -8,6 +8,7 @@ import ( "ds2api/internal/sse" streamengine "ds2api/internal/stream" + "ds2api/internal/util" ) type claudeStreamRuntime struct { @@ -116,6 +117,18 @@ func (s *claudeStreamRuntime) onParsed(parsed sse.LineResult) streamengine.Parse s.text.WriteString(p.Text) if s.bufferToolContent { + if hasUnclosedCodeFence(s.text.String()) { + continue + } + detected := util.ParseToolCalls(s.text.String(), s.toolNames) + if len(detected) > 0 { + s.finalize("tool_use") + return streamengine.ParsedDecision{ + ContentSeen: true, + Stop: true, + StopReason: streamengine.StopReason("tool_use_detected"), + } + } continue } s.closeThinkingBlock() @@ -144,3 +157,7 @@ func (s *claudeStreamRuntime) onParsed(parsed sse.LineResult) streamengine.Parse return streamengine.ParsedDecision{ContentSeen: contentSeen} } + +func hasUnclosedCodeFence(text string) bool { + return strings.Count(text, "```")%2 == 1 +} diff --git a/internal/adapter/openai/message_normalize.go b/internal/adapter/openai/message_normalize.go index 724cb9f..c4f4c4a 100644 --- a/internal/adapter/openai/message_normalize.go +++ b/internal/adapter/openai/message_normalize.go @@ -78,7 +78,7 @@ func formatAssistantToolCallsForPrompt(msg map[string]any, traceID string) strin args = normalizeOpenAIArgumentsForPrompt(fn["arguments"]) } if name == "" { - name = "unknown" + continue } if args == "" { args = normalizeOpenAIArgumentsForPrompt(call["arguments"]) diff --git a/internal/adapter/openai/message_normalize_test.go b/internal/adapter/openai/message_normalize_test.go index ecb3bbd..c9c967d 100644 --- a/internal/adapter/openai/message_normalize_test.go +++ b/internal/adapter/openai/message_normalize_test.go @@ -194,6 +194,29 @@ func TestNormalizeOpenAIMessagesForPrompt_PreservesConcatenatedToolArguments(t * } } + +func TestNormalizeOpenAIMessagesForPrompt_AssistantToolCallsMissingNameAreDropped(t *testing.T) { + raw := []any{ + map[string]any{ + "role": "assistant", + "tool_calls": []any{ + map[string]any{ + "id": "call_missing_name", + "type": "function", + "function": map[string]any{ + "arguments": `{"path":"README.MD"}`, + }, + }, + }, + }, + } + + normalized := normalizeOpenAIMessagesForPrompt(raw, "") + if len(normalized) != 0 { + t.Fatalf("expected nameless assistant tool_calls to be dropped, got %#v", normalized) + } +} + func TestNormalizeOpenAIMessagesForPrompt_AssistantNilContentDoesNotInjectNullLiteral(t *testing.T) { raw := []any{ map[string]any{ diff --git a/internal/adapter/openai/responses_stream_runtime_toolcalls.go b/internal/adapter/openai/responses_stream_runtime_toolcalls.go index 9947cbd..ad354d4 100644 --- a/internal/adapter/openai/responses_stream_runtime_toolcalls.go +++ b/internal/adapter/openai/responses_stream_runtime_toolcalls.go @@ -94,6 +94,16 @@ func (s *responsesStreamRuntime) closeMessageItem() { outputIndex := s.ensureMessageOutputIndex() text := s.visibleText.String() if s.messagePartAdded { + s.sendEvent( + "response.output_text.done", + openaifmt.BuildResponsesTextDonePayload( + s.responseID, + itemID, + outputIndex, + 0, + text, + ), + ) s.sendEvent( "response.content_part.done", openaifmt.BuildResponsesContentPartDonePayload( diff --git a/internal/adapter/openai/responses_stream_test.go b/internal/adapter/openai/responses_stream_test.go index 6186461..f62ff13 100644 --- a/internal/adapter/openai/responses_stream_test.go +++ b/internal/adapter/openai/responses_stream_test.go @@ -226,6 +226,40 @@ func TestHandleResponsesStreamMultiToolCallKeepsNameAndCallIDAligned(t *testing. } } +func TestHandleResponsesStreamEmitsOutputTextDoneBeforeContentPartDone(t *testing.T) { + h := &Handler{} + req := httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + rec := httptest.NewRecorder() + + sseLine := func(v string) string { + b, _ := json.Marshal(map[string]any{ + "p": "response/content", + "v": v, + }) + return "data: " + string(b) + "\n" + } + + streamBody := sseLine("hello") + "data: [DONE]\n" + resp := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(streamBody)), + } + + h.handleResponsesStream(rec, req, resp, "owner-a", "resp_test", "deepseek-chat", "prompt", false, false, nil, util.DefaultToolChoicePolicy(), "") + body := rec.Body.String() + if !strings.Contains(body, "event: response.output_text.done") { + t.Fatalf("expected response.output_text.done payload, body=%s", body) + } + textDoneIdx := strings.Index(body, "event: response.output_text.done") + partDoneIdx := strings.Index(body, "event: response.content_part.done") + if textDoneIdx < 0 || partDoneIdx < 0 { + t.Fatalf("expected output_text.done + content_part.done, body=%s", body) + } + if textDoneIdx > partDoneIdx { + t.Fatalf("expected output_text.done before content_part.done, body=%s", body) + } +} + func TestHandleResponsesStreamOutputTextDeltaCarriesItemIndexes(t *testing.T) { h := &Handler{} req := httptest.NewRequest(http.MethodPost, "/v1/responses", nil) diff --git a/internal/format/openai/render_stream_events.go b/internal/format/openai/render_stream_events.go index dc13231..1e7cd09 100644 --- a/internal/format/openai/render_stream_events.go +++ b/internal/format/openai/render_stream_events.go @@ -71,6 +71,19 @@ func BuildResponsesTextDeltaPayload(responseID, itemID string, outputIndex, cont } } + +func BuildResponsesTextDonePayload(responseID, itemID string, outputIndex, contentIndex int, text string) map[string]any { + return map[string]any{ + "type": "response.output_text.done", + "id": responseID, + "response_id": responseID, + "item_id": itemID, + "output_index": outputIndex, + "content_index": contentIndex, + "text": text, + } +} + func BuildResponsesReasoningDeltaPayload(responseID, delta string) map[string]any { return map[string]any{ "type": "response.reasoning.delta", diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index 1f1fc59..c1b92a8 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -21,22 +21,14 @@ function processToolSieveChunk(state, chunk, toolNames) { } const events = []; - if (Array.isArray(state.pendingToolCalls) && state.pendingToolCalls.length > 0) { - const pending = state.pending || ''; - if (pending.trim() !== '') { - const content = (state.pendingToolRaw || '') + pending; - state.pending = ''; - state.pendingToolRaw = ''; - state.pendingToolCalls = []; - noteText(state, content); - events.push({ type: 'text', text: content }); - } else { - return events; - } - } - // eslint-disable-next-line no-constant-condition while (true) { + if (Array.isArray(state.pendingToolCalls) && state.pendingToolCalls.length > 0) { + events.push({ type: 'tool_calls', calls: state.pendingToolCalls }); + state.pendingToolRaw = ''; + state.pendingToolCalls = []; + continue; + } if (state.capturing) { if (state.pending) { state.capture += state.pending; diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index c3e76bf..53eac8e 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -39,6 +39,9 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa var parsed []ParsedToolCall for _, candidate := range candidates { tc := parseToolCallsPayload(candidate) + if len(tc) == 0 { + tc = parseXMLToolCalls(candidate) + } if len(tc) == 0 { tc = parseMarkupToolCalls(candidate) } @@ -49,7 +52,11 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa } } if len(parsed) == 0 { - return result + parsed = parseXMLToolCalls(text) + if len(parsed) == 0 { + return result + } + result.SawToolCallSyntax = true } calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) @@ -80,6 +87,9 @@ func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) continue } parsed := parseToolCallsPayload(candidate) + if len(parsed) == 0 { + parsed = parseXMLToolCalls(candidate) + } if len(parsed) == 0 { parsed = parseMarkupToolCalls(candidate) } diff --git a/internal/util/toolcalls_parse_markup.go b/internal/util/toolcalls_parse_markup.go new file mode 100644 index 0000000..b7b2908 --- /dev/null +++ b/internal/util/toolcalls_parse_markup.go @@ -0,0 +1,219 @@ +package util + +import ( + "encoding/json" + "encoding/xml" + "regexp" + "strings" +) + +var xmlToolCallPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) +var functionCallPattern = regexp.MustCompile(`(?is)\s*([^<]+?)\s*`) +var functionParamPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) +var antmlFunctionCallPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?function_call[^>]*(?:name|function)="([^"]+)"[^>]*>\s*(.*?)\s*`) +var antmlArgumentPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?argument\s+name="([^"]+)"\s*>\s*(.*?)\s*`) +var antmlParametersPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?parameters\s*>\s*(\{.*?\})\s*`) +var invokeCallPattern = regexp.MustCompile(`(?is)(.*?)`) +var invokeParamPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) + +func parseXMLToolCalls(text string) []ParsedToolCall { + matches := xmlToolCallPattern.FindAllString(text, -1) + out := make([]ParsedToolCall, 0, len(matches)+1) + for _, block := range matches { + call, ok := parseSingleXMLToolCall(block) + if !ok { + continue + } + out = append(out, call) + } + if len(out) > 0 { + return out + } + if call, ok := parseFunctionCallTagStyle(text); ok { + return []ParsedToolCall{call} + } + if calls := parseAntmlFunctionCallStyles(text); len(calls) > 0 { + return calls + } + if call, ok := parseInvokeFunctionCallStyle(text); ok { + return []ParsedToolCall{call} + } + return nil +} + +func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { + inner := strings.TrimSpace(block) + inner = strings.TrimPrefix(inner, "") + inner = strings.TrimSuffix(inner, "") + inner = strings.TrimSpace(inner) + if strings.HasPrefix(inner, "{") { + var payload map[string]any + if err := json.Unmarshal([]byte(inner), &payload); err == nil { + name := strings.TrimSpace(asString(payload["tool"])) + if name == "" { + name = strings.TrimSpace(asString(payload["tool_name"])) + } + if name != "" { + input := map[string]any{} + if params, ok := payload["params"].(map[string]any); ok { + input = params + } else if params, ok := payload["parameters"].(map[string]any); ok { + input = params + } + return ParsedToolCall{Name: name, Input: input}, true + } + } + } + + dec := xml.NewDecoder(strings.NewReader(block)) + name := "" + params := map[string]any{} + inParams := false + inTool := false + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + tag := strings.ToLower(t.Name.Local) + switch tag { + case "tool": + inTool = true + for _, attr := range t.Attr { + if strings.EqualFold(strings.TrimSpace(attr.Name.Local), "name") && strings.TrimSpace(name) == "" { + name = strings.TrimSpace(attr.Value) + } + } + case "parameters": + inParams = true + case "tool_name", "name": + var v string + if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" { + name = strings.TrimSpace(v) + } + default: + if inParams || inTool { + var v string + if err := dec.DecodeElement(&v, &t); err == nil { + params[t.Name.Local] = strings.TrimSpace(v) + } + } + } + case xml.EndElement: + tag := strings.ToLower(t.Name.Local) + if tag == "parameters" { + inParams = false + } + if tag == "tool" { + inTool = false + } + } + } + if strings.TrimSpace(name) == "" { + return ParsedToolCall{}, false + } + return ParsedToolCall{Name: strings.TrimSpace(name), Input: params}, true +} + +func parseFunctionCallTagStyle(text string) (ParsedToolCall, bool) { + m := functionCallPattern.FindStringSubmatch(text) + if len(m) < 2 { + return ParsedToolCall{}, false + } + name := strings.TrimSpace(m[1]) + if name == "" { + return ParsedToolCall{}, false + } + input := map[string]any{} + for _, pm := range functionParamPattern.FindAllStringSubmatch(text, -1) { + if len(pm) < 3 { + continue + } + key := strings.TrimSpace(pm[1]) + val := strings.TrimSpace(pm[2]) + if key != "" { + input[key] = val + } + } + return ParsedToolCall{Name: name, Input: input}, true +} + +func parseAntmlFunctionCallStyles(text string) []ParsedToolCall { + matches := antmlFunctionCallPattern.FindAllStringSubmatch(text, -1) + if len(matches) == 0 { + return nil + } + out := make([]ParsedToolCall, 0, len(matches)) + for _, m := range matches { + if call, ok := parseSingleAntmlFunctionCallMatch(m); ok { + out = append(out, call) + } + } + if len(out) == 0 { + return nil + } + return out +} + +func parseSingleAntmlFunctionCallMatch(m []string) (ParsedToolCall, bool) { + if len(m) < 3 { + return ParsedToolCall{}, false + } + name := strings.TrimSpace(m[1]) + if name == "" { + return ParsedToolCall{}, false + } + body := strings.TrimSpace(m[2]) + input := map[string]any{} + if strings.HasPrefix(body, "{") { + if err := json.Unmarshal([]byte(body), &input); err == nil { + return ParsedToolCall{Name: name, Input: input}, true + } + } + if pm := antmlParametersPattern.FindStringSubmatch(body); len(pm) >= 2 { + if err := json.Unmarshal([]byte(strings.TrimSpace(pm[1])), &input); err == nil { + return ParsedToolCall{Name: name, Input: input}, true + } + } + for _, am := range antmlArgumentPattern.FindAllStringSubmatch(body, -1) { + if len(am) < 3 { + continue + } + k := strings.TrimSpace(am[1]) + v := strings.TrimSpace(am[2]) + if k != "" { + input[k] = v + } + } + return ParsedToolCall{Name: name, Input: input}, true +} + +func parseInvokeFunctionCallStyle(text string) (ParsedToolCall, bool) { + m := invokeCallPattern.FindStringSubmatch(text) + if len(m) < 3 { + return ParsedToolCall{}, false + } + name := strings.TrimSpace(m[1]) + if name == "" { + return ParsedToolCall{}, false + } + input := map[string]any{} + for _, pm := range invokeParamPattern.FindAllStringSubmatch(m[2], -1) { + if len(pm) < 3 { + continue + } + k := strings.TrimSpace(pm[1]) + v := strings.TrimSpace(pm[2]) + if k != "" { + input[k] = v + } + } + return ParsedToolCall{Name: name, Input: input}, true +} + +func asString(v any) string { + s, _ := v.(string) + return s +} diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index e830092..3ace015 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -138,6 +138,140 @@ func TestParseToolCallsAllowsPunctuationVariantToolName(t *testing.T) { } } +func TestParseToolCallsSupportsClaudeXMLToolCall(t *testing.T) { + text := `Bashpwdshow cwd` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsDetailedMarksXMLToolCallSyntax(t *testing.T) { + text := `Bashpwd` + res := ParseToolCallsDetailed(text, []string{"bash"}) + if !res.SawToolCallSyntax { + t.Fatalf("expected SawToolCallSyntax=true, got %#v", res) + } + if len(res.Calls) != 1 { + t.Fatalf("expected one parsed call, got %#v", res) + } +} + +func TestParseToolCallsSupportsClaudeXMLJSONToolCall(t *testing.T) { + text := `{"tool":"Bash","params":{"command":"pwd","description":"show cwd"}}` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsFunctionCallTagStyle(t *testing.T) { + text := `Bashls -lalist` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "ls -la" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsAntmlFunctionCallStyle(t *testing.T) { + text := `{"command":"pwd","description":"x"}` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsAntmlArgumentStyle(t *testing.T) { + text := `pwdx` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsInvokeFunctionCallStyle(t *testing.T) { + text := `pwdd` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsNestedToolTagStyle(t *testing.T) { + text := `pwdshow cwd` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsAntmlFunctionAttributeWithParametersTag(t *testing.T) { + text := `{"command":"pwd"}` + calls := ParseToolCalls(text, []string{"bash"}) + if len(calls) != 1 { + t.Fatalf("expected 1 call, got %#v", calls) + } + if calls[0].Name != "bash" { + t.Fatalf("expected canonical tool name bash, got %q", calls[0].Name) + } + if calls[0].Input["command"] != "pwd" { + t.Fatalf("expected command argument, got %#v", calls[0].Input) + } +} + +func TestParseToolCallsSupportsMultipleAntmlFunctionCalls(t *testing.T) { + text := `{"command":"pwd"}{"file_path":"README.md"}` + calls := ParseToolCalls(text, []string{"bash", "read"}) + if len(calls) != 2 { + t.Fatalf("expected 2 calls, got %#v", calls) + } + if calls[0].Name != "bash" || calls[1].Name != "read" { + t.Fatalf("expected canonical names [bash read], got %#v", calls) + } +} + func TestParseToolCallsDoesNotAcceptMismatchedMarkupTags(t *testing.T) { text := `read_file{"path":"README.md"}` calls := ParseToolCalls(text, []string{"read_file"}) diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index 20c00b8..f71279c 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -109,7 +109,23 @@ test('parseStandaloneToolCalls ignores fenced code block tool_call examples', () assert.equal(calls.length, 0); }); -test('sieve keeps late key convergence payload as plain text in strict mode', () => { + +test('sieve emits tool_calls in the same chunk processing tick once payload is complete', () => { + const state = createToolSieveState(); + const first = processToolSieveChunk(state, '{"', ['read_file']); + const second = processToolSieveChunk( + state, + 'tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}', + ['read_file'], + ); + const firstCalls = first.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + const secondCalls = second.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(firstCalls.length, 0); + assert.equal(secondCalls.length, 1); + assert.equal(secondCalls[0].name, 'read_file'); +}); + +test('sieve emits tool_calls when late key convergence forms a complete payload', () => { const events = runSieve( [ '{"', @@ -119,12 +135,11 @@ test('sieve keeps late key convergence payload as plain text in strict mode', () ['read_file'], ); const leakedText = collectText(events); - const hasToolCall = events.some((evt) => evt.type === 'tool_calls' && Array.isArray(evt.calls) && evt.calls.length > 0); - const hasToolDelta = events.some((evt) => evt.type === 'tool_call_deltas' && Array.isArray(evt.deltas) && evt.deltas.length > 0); - assert.equal(hasToolCall || hasToolDelta, false); - assert.equal(leakedText.includes('{'), true); - assert.equal(leakedText.toLowerCase().includes('tool_calls'), true); + const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); + assert.equal(finalCalls.length, 1); + assert.equal(finalCalls[0].name, 'read_file'); assert.equal(leakedText.includes('后置正文C。'), true); + assert.equal(leakedText.toLowerCase().includes('tool_calls'), false); }); test('sieve keeps embedded invalid tool-like json as normal text to avoid stream stalls', () => { From 11b2f24fc27a8cbb633dc1b25bc64d2ce816dc30 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Sun, 8 Mar 2026 02:30:12 +0800 Subject: [PATCH 4/6] Merge origin/dev into PR branch and resolve toolcall parser conflicts --- internal/util/toolcalls_parse_markup.go | 26 +++++++++++++++++++++++++ internal/util/toolcalls_test.go | 1 - 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/util/toolcalls_parse_markup.go b/internal/util/toolcalls_parse_markup.go index b7b2908..355a464 100644 --- a/internal/util/toolcalls_parse_markup.go +++ b/internal/util/toolcalls_parse_markup.go @@ -15,6 +15,7 @@ var antmlArgumentPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?argument\s+ var antmlParametersPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?parameters\s*>\s*(\{.*?\})\s*`) var invokeCallPattern = regexp.MustCompile(`(?is)(.*?)`) var invokeParamPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) +var invokeArgumentPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) func parseXMLToolCalls(text string) []ParsedToolCall { matches := xmlToolCallPattern.FindAllString(text, -1) @@ -111,6 +112,14 @@ func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { } } } + if fallback := parseMarkupSingleToolCall("", inner); fallback.Name != "" { + if strings.TrimSpace(name) == "" { + name = fallback.Name + } + if len(params) == 0 && strings.EqualFold(strings.TrimSpace(fallback.Name), strings.TrimSpace(name)) { + params = fallback.Input + } + } if strings.TrimSpace(name) == "" { return ParsedToolCall{}, false } @@ -210,6 +219,23 @@ func parseInvokeFunctionCallStyle(text string) (ParsedToolCall, bool) { input[k] = v } } + for _, am := range invokeArgumentPattern.FindAllStringSubmatch(m[2], -1) { + if len(am) < 3 { + continue + } + key := strings.TrimSpace(am[1]) + raw := strings.TrimSpace(am[2]) + if raw == "" { + continue + } + if key != "" { + input[key] = raw + continue + } + for k, v := range parseToolCallInput(raw) { + input[k] = v + } + } return ParsedToolCall{Name: name, Input: input}, true } diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index 3ace015..fb5c246 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -271,7 +271,6 @@ func TestParseToolCallsSupportsMultipleAntmlFunctionCalls(t *testing.T) { t.Fatalf("expected canonical names [bash read], got %#v", calls) } } - func TestParseToolCallsDoesNotAcceptMismatchedMarkupTags(t *testing.T) { text := `read_file{"path":"README.md"}` calls := ParseToolCalls(text, []string{"read_file"}) From 286d2667230d7a10f7694aa81fdc781056ca165d Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Sun, 8 Mar 2026 02:38:29 +0800 Subject: [PATCH 5/6] Revert "Resolve PR #82 merge conflicts and restore tool-call parsing (invoke/argument and XML arguments)" --- internal/util/toolcalls_parse_markup.go | 26 ------------------------- internal/util/toolcalls_test.go | 1 + 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/internal/util/toolcalls_parse_markup.go b/internal/util/toolcalls_parse_markup.go index 355a464..b7b2908 100644 --- a/internal/util/toolcalls_parse_markup.go +++ b/internal/util/toolcalls_parse_markup.go @@ -15,7 +15,6 @@ var antmlArgumentPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?argument\s+ var antmlParametersPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?parameters\s*>\s*(\{.*?\})\s*`) var invokeCallPattern = regexp.MustCompile(`(?is)(.*?)`) var invokeParamPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) -var invokeArgumentPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) func parseXMLToolCalls(text string) []ParsedToolCall { matches := xmlToolCallPattern.FindAllString(text, -1) @@ -112,14 +111,6 @@ func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { } } } - if fallback := parseMarkupSingleToolCall("", inner); fallback.Name != "" { - if strings.TrimSpace(name) == "" { - name = fallback.Name - } - if len(params) == 0 && strings.EqualFold(strings.TrimSpace(fallback.Name), strings.TrimSpace(name)) { - params = fallback.Input - } - } if strings.TrimSpace(name) == "" { return ParsedToolCall{}, false } @@ -219,23 +210,6 @@ func parseInvokeFunctionCallStyle(text string) (ParsedToolCall, bool) { input[k] = v } } - for _, am := range invokeArgumentPattern.FindAllStringSubmatch(m[2], -1) { - if len(am) < 3 { - continue - } - key := strings.TrimSpace(am[1]) - raw := strings.TrimSpace(am[2]) - if raw == "" { - continue - } - if key != "" { - input[key] = raw - continue - } - for k, v := range parseToolCallInput(raw) { - input[k] = v - } - } return ParsedToolCall{Name: name, Input: input}, true } diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index fb5c246..3ace015 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -271,6 +271,7 @@ func TestParseToolCallsSupportsMultipleAntmlFunctionCalls(t *testing.T) { t.Fatalf("expected canonical names [bash read], got %#v", calls) } } + func TestParseToolCallsDoesNotAcceptMismatchedMarkupTags(t *testing.T) { text := `read_file{"path":"README.md"}` calls := ParseToolCalls(text, []string{"read_file"}) From ea4bd1e4839d10618150ce16ac5298c1761987a4 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Sun, 8 Mar 2026 13:16:12 +0800 Subject: [PATCH 6/6] fix: parse invoke/tool_call arguments in xml compatibility paths --- internal/util/toolcalls_parse_markup.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/util/toolcalls_parse_markup.go b/internal/util/toolcalls_parse_markup.go index b7b2908..e2eff83 100644 --- a/internal/util/toolcalls_parse_markup.go +++ b/internal/util/toolcalls_parse_markup.go @@ -93,6 +93,15 @@ func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" { name = strings.TrimSpace(v) } + case "input", "arguments", "argument", "args", "params": + var v string + if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" { + if parsed := parseToolCallInput(strings.TrimSpace(v)); len(parsed) > 0 { + for k, vv := range parsed { + params[k] = vv + } + } + } default: if inParams || inTool { var v string @@ -210,6 +219,13 @@ func parseInvokeFunctionCallStyle(text string) (ParsedToolCall, bool) { input[k] = v } } + if len(input) == 0 { + if argsRaw := findMarkupTagValue(m[2], toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); argsRaw != "" { + input = parseMarkupInput(argsRaw) + } else if kv := parseMarkupKVObject(m[2]); len(kv) > 0 { + input = kv + } + } return ParsedToolCall{Name: name, Input: input}, true }