diff --git a/README.MD b/README.MD index 636e693..6b9383a 100644 --- a/README.MD +++ b/README.MD @@ -363,8 +363,6 @@ cp opencode.json.example opencode.json 3. 未在 `tools` 声明中的工具名会被严格拒绝，不会下发为有效 tool call 4. `responses` 支持并执行 `tool_choice`（`auto`/`none`/`required`/强制函数）；`required` 违规时非流式返回 `422`，流式返回 `response.failed` 5. 仅在通过策略校验后才会发出有效工具调用事件，避免错误工具名进入客户端执行链 -6. strict 模式下采用“可解析即拦截”：即使 tool JSON 前后混有 prose，只要结构可提取仍会拦截 tool_calls，剩余文本继续透传 -7. 当参数字符串无法可靠修复为对象时，会保留 `{"_raw":"..."}` 回退，避免 silent corruption ## 本地开发抓包工具 diff --git a/TESTING.md b/TESTING.md index bf821fe..8d1a309 100644 --- a/TESTING.md +++ b/TESTING.md @@ -200,13 +200,6 @@ go test -v -run 'TestParseToolCalls|TestRepair' ./internal/util/ # 2. 查看测试输出中的详细调试信息 go test -v -run TestParseToolCallsWithDeepSeekHallucination ./internal/util/ 2>&1 -# 2.1 strict 模式（Go/JS）语义对齐检查：混合 prose + tool JSON 仍可拦截 -node --test tests/node/stream-tool-sieve.test.js - -# 2.2 Windows 路径与文本换行语义回归 -go test -v -run TestParseToolCallsWithInvalidBackslashes ./internal/util/ -go test -v -run TestParseToolCallsWithPathEscapesAndTextNewlines ./internal/util/ - # 3. 检查具体测试用例的修复效果 # 测试用例位于 internal/util/toolcalls_test.go，包含： # - TestParseToolCallsWithDeepSeekHallucination: DeepSeek 典型幻觉输出 diff --git a/internal/adapter/claude/handler_stream_test.go b/internal/adapter/claude/handler_stream_test.go index dda425a..77e62c8 100644 --- a/internal/adapter/claude/handler_stream_test.go +++ b/internal/adapter/claude/handler_stream_test.go @@ -358,7 +358,7 @@ func TestHandleClaudeStreamRealtimeToolSafetyAcrossStructuredFormats(t *testing. } } -func TestHandleClaudeStreamRealtimeDoesNotStopOnUnclosedFencedToolExample(t *testing.T) { +func TestHandleClaudeStreamRealtimePromotesUnclosedFencedToolExample(t *testing.T) { h := &Handler{} resp := makeClaudeSSEHTTPResponse( "data: {\"p\":\"response/content\",\"v\":\"Here is an example:\\n```json\\n{\\\"tool_calls\\\":[{\\\"name\\\":\\\"Bash\\\",\\\"input\\\":{\\\"command\\\":\\\"pwd\\\"}}]}\"}", @@ -371,22 +371,27 @@ func TestHandleClaudeStreamRealtimeDoesNotStopOnUnclosedFencedToolExample(t *tes h.handleClaudeStreamRealtime(rec, req, resp, "claude-sonnet-4-5", []any{map[string]any{"role": "user", "content": "show example only"}}, false, false, []string{"Bash"}) frames := parseClaudeFrames(t, rec.Body.String()) + foundToolUse := false for _, f := range findClaudeFrames(frames, "content_block_start") { contentBlock, _ := f.Payload["content_block"].(map[string]any) if contentBlock["type"] == "tool_use" { - t.Fatalf("unexpected tool_use for fenced example, body=%s", rec.Body.String()) - } - } - - foundEndTurn := false - for _, f := range findClaudeFrames(frames, "message_delta") { - delta, _ := f.Payload["delta"].(map[string]any) - if delta["stop_reason"] == "end_turn" { - foundEndTurn = true + foundToolUse = true break } } - if !foundEndTurn { - t.Fatalf("expected stop_reason=end_turn, body=%s", rec.Body.String()) + if !foundToolUse { + t.Fatalf("expected tool_use for fenced example, body=%s", rec.Body.String()) + } + + foundToolStop := false + for _, f := range findClaudeFrames(frames, "message_delta") { + delta, _ := f.Payload["delta"].(map[string]any) + if delta["stop_reason"] == "tool_use" { + foundToolStop = true + break + } + } + if !foundToolStop { + t.Fatalf("expected stop_reason=tool_use, body=%s", rec.Body.String()) } } diff --git a/internal/adapter/openai/handler_toolcall_test.go b/internal/adapter/openai/handler_toolcall_test.go index 0d85afd..ef22803 100644 --- a/internal/adapter/openai/handler_toolcall_test.go +++ b/internal/adapter/openai/handler_toolcall_test.go @@ -211,7 +211,7 @@ func TestHandleNonStreamUnknownToolNotIntercepted(t *testing.T) { } } -func TestHandleNonStreamEmbeddedToolCallExampleRemainsText(t *testing.T) { +func TestHandleNonStreamEmbeddedToolCallExamplePromotesToolCall(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( `data: {"p":"response/content","v":"下面是示例："}`, @@ -229,20 +229,21 @@ func TestHandleNonStreamEmbeddedToolCallExampleRemainsText(t *testing.T) { out := decodeJSONBody(t, rec.Body.String()) choices, _ := out["choices"].([]any) choice, _ := choices[0].(map[string]any) - if choice["finish_reason"] != "stop" { - t.Fatalf("expected finish_reason=stop, got %#v", choice["finish_reason"]) + if choice["finish_reason"] != "tool_calls" { + t.Fatalf("expected finish_reason=tool_calls, got %#v", choice["finish_reason"]) } msg, _ := choice["message"].(map[string]any) - if _, ok := msg["tool_calls"]; ok { - t.Fatalf("did not expect tool_calls field for embedded example: %#v", msg["tool_calls"]) + toolCalls, _ := msg["tool_calls"].([]any) + if len(toolCalls) != 1 { + t.Fatalf("expected one tool_call field for embedded example: %#v", msg["tool_calls"]) } content, _ := msg["content"].(string) - if !strings.Contains(content, "下面是示例：") || !strings.Contains(content, "请勿执行。") || !strings.Contains(content, `"tool_calls"`) { - t.Fatalf("expected embedded example to remain plain text, got %#v", content) + if strings.Contains(content, `"tool_calls"`) { + t.Fatalf("expected raw tool_calls json stripped from content, got %#v", content) } } -func TestHandleNonStreamFencedToolCallExampleNotIntercepted(t *testing.T) { +func TestHandleNonStreamFencedToolCallExamplePromotesToolCall(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( "data: {\"p\":\"response/content\",\"v\":\"```json\\n{\\\"tool_calls\\\":[{\\\"name\\\":\\\"search\\\",\\\"input\\\":{\\\"q\\\":\\\"go\\\"}}]}\\n```\"}", @@ -258,16 +259,17 @@ func TestHandleNonStreamFencedToolCallExampleNotIntercepted(t *testing.T) { out := decodeJSONBody(t, rec.Body.String()) choices, _ := out["choices"].([]any) choice, _ := choices[0].(map[string]any) - if choice["finish_reason"] != "stop" { - t.Fatalf("expected finish_reason=stop, got %#v", choice["finish_reason"]) + if choice["finish_reason"] != "tool_calls" { + t.Fatalf("expected finish_reason=tool_calls, got %#v", choice["finish_reason"]) } msg, _ := choice["message"].(map[string]any) - if _, ok := msg["tool_calls"]; ok { - t.Fatalf("did not expect tool_calls field for fenced example: %#v", msg["tool_calls"]) + toolCalls, _ := msg["tool_calls"].([]any) + if len(toolCalls) != 1 { + t.Fatalf("expected one tool_call field for fenced example: %#v", msg["tool_calls"]) } content, _ := msg["content"].(string) - if !strings.Contains(content, "```json") || !strings.Contains(content, `"tool_calls"`) { - t.Fatalf("expected fenced tool example to pass through as text, got %q", content) + if strings.Contains(content, `"tool_calls"`) { + t.Fatalf("expected raw tool_calls json stripped from content, got %q", content) } } @@ -615,7 +617,7 @@ func TestHandleStreamToolCallWithSameChunkTrailingTextRemainsText(t *testing.T) } } -func TestHandleStreamFencedToolCallSnippetRemainsText(t *testing.T) { +func TestHandleStreamFencedToolCallSnippetPromotesToolCall(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( fmt.Sprintf(`data: {"p":"response/content","v":%q}`, "下面是调用示例：\n```json\n"), @@ -631,8 +633,8 @@ func TestHandleStreamFencedToolCallSnippetRemainsText(t *testing.T) { if !done { t.Fatalf("expected [DONE], body=%s", rec.Body.String()) } - if streamHasToolCallsDelta(frames) { - t.Fatalf("did not expect tool_calls delta for fenced snippet, body=%s", rec.Body.String()) + if !streamHasToolCallsDelta(frames) { + t.Fatalf("expected tool_calls delta for fenced snippet, body=%s", rec.Body.String()) } content := strings.Builder{} for _, frame := range frames { @@ -646,11 +648,11 @@ func TestHandleStreamFencedToolCallSnippetRemainsText(t *testing.T) { } } got := content.String() - if !strings.Contains(got, "```json") || !strings.Contains(strings.ToLower(got), "tool_calls") { - t.Fatalf("expected fenced tool snippet in content, got=%q", got) + if strings.Contains(strings.ToLower(got), "tool_calls") { + t.Fatalf("expected raw fenced tool_calls snippet stripped from content, got=%q", got) } - if streamFinishReason(frames) != "stop" { - t.Fatalf("expected finish_reason=stop, body=%s", rec.Body.String()) + if streamFinishReason(frames) != "tool_calls" { + t.Fatalf("expected finish_reason=tool_calls, body=%s", rec.Body.String()) } } diff --git a/internal/adapter/openai/responses_stream_test.go b/internal/adapter/openai/responses_stream_test.go index f62ff13..7d15ede 100644 --- a/internal/adapter/openai/responses_stream_test.go +++ b/internal/adapter/openai/responses_stream_test.go @@ -297,7 +297,7 @@ func TestHandleResponsesStreamOutputTextDeltaCarriesItemIndexes(t *testing.T) { } } -func TestHandleResponsesStreamThinkingAndMixedToolExampleRemainMessageOnly(t *testing.T) { +func TestHandleResponsesStreamThinkingAndMixedToolExampleEmitsFunctionCall(t *testing.T) { h := &Handler{} req := httptest.NewRequest(http.MethodPost, "/v1/responses", nil) rec := httptest.NewRecorder() @@ -333,6 +333,7 @@ func TestHandleResponsesStreamThinkingAndMixedToolExampleRemainMessageOnly(t *te responseObj, _ := completedPayload["response"].(map[string]any) output, _ := responseObj["output"].([]any) hasMessage := false + hasFunctionCall := false for _, item := range output { m, _ := item.(map[string]any) if m == nil { @@ -342,12 +343,15 @@ func TestHandleResponsesStreamThinkingAndMixedToolExampleRemainMessageOnly(t *te hasMessage = true } if asString(m["type"]) == "function_call" { - t.Fatalf("did not expect function_call output for mixed prose tool example, output=%#v", output) + hasFunctionCall = true } } if !hasMessage { t.Fatalf("expected message output for mixed prose tool example, output=%#v", output) } + if !hasFunctionCall { + t.Fatalf("expected function_call output for mixed prose tool example, output=%#v", output) + } } func TestHandleResponsesStreamToolChoiceNoneRejectsFunctionCall(t *testing.T) { diff --git a/internal/adapter/openai/stream_status_test.go b/internal/adapter/openai/stream_status_test.go index 033dc37..c76d881 100644 --- a/internal/adapter/openai/stream_status_test.go +++ b/internal/adapter/openai/stream_status_test.go @@ -171,15 +171,15 @@ func TestResponsesNonStreamMixedProseToolPayloadHandlerPath(t *testing.T) { t.Fatalf("decode response failed: %v body=%s", err, rec.Body.String()) } outputText, _ := out["output_text"].(string) - if outputText == "" { - t.Fatalf("expected output_text preserved for mixed prose payload") + if outputText != "" { + t.Fatalf("expected output_text hidden for mixed prose tool payload, got %q", outputText) } output, _ := out["output"].([]any) if len(output) != 1 { t.Fatalf("expected one output item, got %#v", output) } first, _ := output[0].(map[string]any) - if first["type"] != "message" { - t.Fatalf("expected message output item, got %#v", output) + if first["type"] != "function_call" { + t.Fatalf("expected function_call output item, got %#v", output) } } diff --git a/internal/adapter/openai/tool_sieve_core.go b/internal/adapter/openai/tool_sieve_core.go index 72628e9..ca2223a 100644 --- a/internal/adapter/openai/tool_sieve_core.go +++ b/internal/adapter/openai/tool_sieve_core.go @@ -168,36 +168,21 @@ func findToolSegmentStart(s string) int { } lower := strings.ToLower(s) keywords := []string{"tool_calls", "function.name:", "[tool_call_history]"} - offset := 0 - for { - bestKeyIdx := -1 - matchedKeyword := "" - - for _, kw := range keywords { - idx := strings.Index(lower[offset:], kw) - if idx >= 0 { - absIdx := offset + idx - if bestKeyIdx < 0 || absIdx < bestKeyIdx { - bestKeyIdx = absIdx - matchedKeyword = kw - } - } + bestKeyIdx := -1 + for _, kw := range keywords { + idx := strings.Index(lower, kw) + if idx >= 0 && (bestKeyIdx < 0 || idx < bestKeyIdx) { + bestKeyIdx = idx } - - if bestKeyIdx < 0 { - return -1 - } - - keyIdx := bestKeyIdx - start := strings.LastIndex(s[:keyIdx], "{") - if start < 0 { - start = keyIdx - } - if !insideCodeFence(s[:start]) { - return start - } - offset = keyIdx + len(matchedKeyword) } + if bestKeyIdx < 0 { + return -1 + } + start := strings.LastIndex(s[:bestKeyIdx], "{") + if start < 0 { + start = bestKeyIdx + } + return start } func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix string, calls []util.ParsedToolCall, suffix string, ready bool) { @@ -229,9 +214,6 @@ func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix } prefixPart := captured[:start] suffixPart := captured[end:] - if insideCodeFence(state.recentTextTail + prefixPart) { - return captured, nil, "", true - } parsed := util.ParseStandaloneToolCallsDetailed(obj, toolNames) if len(parsed.Calls) == 0 { if parsed.SawToolCallSyntax && parsed.RejectedByPolicy { diff --git a/internal/adapter/openai/tool_sieve_incremental.go b/internal/adapter/openai/tool_sieve_incremental.go index ad0f901..d0d7842 100644 --- a/internal/adapter/openai/tool_sieve_incremental.go +++ b/internal/adapter/openai/tool_sieve_incremental.go @@ -19,9 +19,6 @@ func buildIncrementalToolDeltas(state *toolStreamSieveState) []toolCallDelta { if start < 0 { return nil } - if insideCodeFence(state.recentTextTail + captured[:start]) { - return nil - } certainSingle, hasMultiple := classifyToolCallsIncrementalSafety(captured, keyIdx) if hasMultiple { state.disableDeltas = true diff --git a/internal/format/openai/render_test.go b/internal/format/openai/render_test.go index 7a9d897..952d0ef 100644 --- a/internal/format/openai/render_test.go +++ b/internal/format/openai/render_test.go @@ -45,7 +45,7 @@ func TestBuildResponseObjectToolCallsFollowChatShape(t *testing.T) { } } -func TestBuildResponseObjectTreatsMixedProseToolPayloadAsText(t *testing.T) { +func TestBuildResponseObjectPromotesMixedProseToolPayloadToFunctionCall(t *testing.T) { obj := BuildResponseObject( "resp_test", "gpt-4o", @@ -56,20 +56,20 @@ func TestBuildResponseObjectTreatsMixedProseToolPayloadAsText(t *testing.T) { ) outputText, _ := obj["output_text"].(string) - if outputText == "" { - t.Fatalf("expected output_text preserved for mixed prose payload") + if outputText != "" { + t.Fatalf("expected output_text hidden for mixed prose tool payload, got %q", outputText) } output, _ := obj["output"].([]any) if len(output) != 1 { - t.Fatalf("expected one message output item, got %#v", obj["output"]) + t.Fatalf("expected one function_call output item, got %#v", obj["output"]) } first, _ := output[0].(map[string]any) - if first["type"] != "message" { - t.Fatalf("expected message output type, got %#v", first["type"]) + if first["type"] != "function_call" { + t.Fatalf("expected function_call output type, got %#v", first["type"]) } } -func TestBuildResponseObjectFencedToolPayloadRemainsText(t *testing.T) { +func TestBuildResponseObjectPromotesFencedToolPayloadToFunctionCall(t *testing.T) { obj := BuildResponseObject( "resp_test", "gpt-4o", @@ -80,16 +80,16 @@ func TestBuildResponseObjectFencedToolPayloadRemainsText(t *testing.T) { ) outputText, _ := obj["output_text"].(string) - if outputText == "" { - t.Fatalf("expected output_text preserved for fenced example") + if outputText != "" { + t.Fatalf("expected output_text hidden for fenced tool payload, got %q", outputText) } output, _ := obj["output"].([]any) if len(output) != 1 { - t.Fatalf("expected one message output item, got %#v", obj["output"]) + t.Fatalf("expected one function_call output item, got %#v", obj["output"]) } first, _ := output[0].(map[string]any) - if first["type"] != "message" { - t.Fatalf("expected message output type, got %#v", first["type"]) + if first["type"] != "function_call" { + t.Fatalf("expected function_call output type, got %#v", first["type"]) } } diff --git a/internal/js/helpers/stream-tool-sieve/parse.js b/internal/js/helpers/stream-tool-sieve/parse.js index 6e6ff7d..22d11d1 100644 --- a/internal/js/helpers/stream-tool-sieve/parse.js +++ b/internal/js/helpers/stream-tool-sieve/parse.js @@ -2,10 +2,8 @@ const { toStringSafe, - looksLikeToolExampleContext, } = require('./state'); const { - stripFencedCodeBlocks, buildToolCallCandidates, parseToolCallsPayload, parseMarkupToolCalls, @@ -38,16 +36,13 @@ function parseToolCalls(text, toolNames) { function parseToolCallsDetailed(text, toolNames) { const result = emptyParseResult(); - if (!toStringSafe(text)) { + const normalized = toStringSafe(text); + if (!normalized) { return result; } - const sanitized = stripFencedCodeBlocks(text); - if (!toStringSafe(sanitized)) { - return result; - } - result.sawToolCallSyntax = looksLikeToolCallSyntax(sanitized); + result.sawToolCallSyntax = looksLikeToolCallSyntax(normalized); - const candidates = buildToolCallCandidates(sanitized); + const candidates = buildToolCallCandidates(normalized); let parsed = []; for (const c of candidates) { parsed = parseToolCallsPayload(c); @@ -63,9 +58,9 @@ function parseToolCallsDetailed(text, toolNames) { } } if (parsed.length === 0) { - parsed = parseMarkupToolCalls(sanitized); + parsed = parseMarkupToolCalls(normalized); if (parsed.length === 0) { - parsed = parseTextKVToolCalls(sanitized); + parsed = parseTextKVToolCalls(normalized); if (parsed.length === 0) { return result; } @@ -90,22 +85,29 @@ function parseStandaloneToolCallsDetailed(text, toolNames) { if (!trimmed) { return result; } - if (trimmed.includes('```')) { - return result; - } - if (looksLikeToolExampleContext(trimmed)) { - return result; - } result.sawToolCallSyntax = looksLikeToolCallSyntax(trimmed); - let parsed = parseToolCallsPayload(trimmed); + const candidates = buildToolCallCandidates(trimmed); + let parsed = []; + for (const c of candidates) { + parsed = parseToolCallsPayload(c); + if (parsed.length === 0) { + parsed = parseMarkupToolCalls(c); + } + if (parsed.length === 0) { + parsed = parseTextKVToolCalls(c); + } + if (parsed.length > 0) { + break; + } + } if (parsed.length === 0) { parsed = parseMarkupToolCalls(trimmed); - } - if (parsed.length === 0) { - parsed = parseTextKVToolCalls(trimmed); - } - if (parsed.length === 0) { - return result; + if (parsed.length === 0) { + parsed = parseTextKVToolCalls(trimmed); + if (parsed.length === 0) { + return result; + } + } } result.sawToolCallSyntax = true; diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 82bb079..3880c2e 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -16,7 +16,6 @@ type ToolCallParseResult struct { RejectedByPolicy bool RejectedToolNames []string } - func ParseToolCalls(text string, availableToolNames []string) []ParsedToolCall { return ParseToolCallsDetailed(text, availableToolNames).Calls } @@ -26,10 +25,6 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa if strings.TrimSpace(text) == "" { return result } - text = stripFencedCodeBlocks(text) - if strings.TrimSpace(text) == "" { - return result - } result.SawToolCallSyntax = looksLikeToolCallSyntax(text) candidates := buildToolCallCandidates(text) @@ -68,7 +63,6 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0 return result } - func ParseStandaloneToolCalls(text string, availableToolNames []string) []ParsedToolCall { return ParseStandaloneToolCallsDetailed(text, availableToolNames).Calls } @@ -79,25 +73,37 @@ func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) if trimmed == "" { return result } - if looksLikeToolExampleContext(trimmed) { - return result - } result.SawToolCallSyntax = looksLikeToolCallSyntax(trimmed) - - parsed := parseToolCallsPayload(trimmed) + candidates := buildToolCallCandidates(trimmed) + var parsed []ParsedToolCall + for _, candidate := range candidates { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + parsed = parseToolCallsPayload(candidate) + if len(parsed) == 0 { + parsed = parseXMLToolCalls(candidate) + } + if len(parsed) == 0 { + parsed = parseMarkupToolCalls(candidate) + } + if len(parsed) == 0 { + parsed = parseTextKVToolCalls(candidate) + } + if len(parsed) > 0 { + break + } + } if len(parsed) == 0 { parsed = parseXMLToolCalls(trimmed) + if len(parsed) == 0 { + parsed = parseTextKVToolCalls(trimmed) + if len(parsed) == 0 { + return result + } + } } - if len(parsed) == 0 { - parsed = parseMarkupToolCalls(trimmed) - } - if len(parsed) == 0 { - parsed = parseTextKVToolCalls(trimmed) - } - if len(parsed) == 0 { - return result - } - result.SawToolCallSyntax = true calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) result.Calls = calls @@ -135,7 +141,6 @@ func filterToolCallsDetailed(parsed []ParsedToolCall, availableToolNames []strin } return nil, rejected } - out := make([]ParsedToolCall, 0, len(parsed)) rejectedSet := map[string]struct{}{} rejected := make([]string, 0) @@ -164,6 +169,31 @@ func resolveAllowedToolName(name string, allowed map[string]struct{}, allowedCan return resolveAllowedToolNameWithLooseMatch(name, allowed, allowedCanonical) } +func parseToolCallsPayload(payload string) []ParsedToolCall { + var decoded any + if err := json.Unmarshal([]byte(payload), &decoded); err != nil { + // Try to repair backslashes first! Because LLMs often mix these two problems. + repaired := repairInvalidJSONBackslashes(payload) + // Try loose repair on top of that + repaired = RepairLooseJSON(repaired) + if err := json.Unmarshal([]byte(repaired), &decoded); err != nil { + return nil + } + } + switch v := decoded.(type) { + case map[string]any: + if tc, ok := v["tool_calls"]; ok { + return parseToolCallList(tc) + } + if parsed, ok := parseToolCallItem(v); ok { + return []ParsedToolCall{parsed} + } + case []any: + return parseToolCallList(v) + } + return nil +} + func looksLikeToolCallSyntax(text string) bool { lower := strings.ToLower(text) return strings.Contains(lower, "tool_calls") || diff --git a/internal/util/toolcalls_parse_payload.go b/internal/util/toolcalls_parse_payload.go deleted file mode 100644 index 5534e10..0000000 --- a/internal/util/toolcalls_parse_payload.go +++ /dev/null @@ -1,185 +0,0 @@ -package util - -import ( - "encoding/json" - "strings" -) - -func parseToolCallsPayload(payload string) []ParsedToolCall { - var decoded any - if err := json.Unmarshal([]byte(payload), &decoded); err != nil { - repaired := repairInvalidJSONBackslashesWithPathContext(payload) - repaired = RepairLooseJSON(repaired) - if err := json.Unmarshal([]byte(repaired), &decoded); err != nil { - return nil - } - } - - switch v := decoded.(type) { - case map[string]any: - if tc, ok := v["tool_calls"]; ok { - return parseToolCallList(tc) - } - if parsed, ok := parseToolCallItem(v); ok { - return []ParsedToolCall{parsed} - } - case []any: - return parseToolCallList(v) - } - return nil -} - -func parseToolCallList(v any) []ParsedToolCall { - items, ok := v.([]any) - if !ok { - return nil - } - out := make([]ParsedToolCall, 0, len(items)) - for _, item := range items { - m, ok := item.(map[string]any) - if !ok { - continue - } - if tc, ok := parseToolCallItem(m); ok { - out = append(out, tc) - } - } - if len(out) == 0 { - return nil - } - return out -} - -func parseToolCallItem(m map[string]any) (ParsedToolCall, bool) { - name, _ := m["name"].(string) - inputRaw, hasInput := m["input"] - - if fn, ok := m["function"].(map[string]any); ok { - if name == "" { - name, _ = fn["name"].(string) - } - if !hasInput { - if v, ok := fn["arguments"]; ok { - inputRaw = v - hasInput = true - } - } - } - if !hasInput { - for _, key := range []string{"arguments", "args", "parameters", "params"} { - if v, ok := m[key]; ok { - inputRaw = v - hasInput = true - break - } - } - } - if strings.TrimSpace(name) == "" { - return ParsedToolCall{}, false - } - return ParsedToolCall{ - Name: strings.TrimSpace(name), - Input: parseToolCallInput(inputRaw), - }, true -} - -func parseToolCallInput(v any) map[string]any { - switch x := v.(type) { - case nil: - return map[string]any{} - case map[string]any: - return x - case string: - raw := strings.TrimSpace(x) - if raw == "" { - return map[string]any{} - } - - if parsed := decodeJSONObject(raw); parsed != nil { - if hasSuspiciousPathControlChars(parsed) { - repaired := repairInvalidJSONBackslashesWithPathContext(raw) - if repaired != raw { - if reparsed := decodeJSONObject(repaired); reparsed != nil { - return reparsed - } - } - } - return parsed - } - - repaired := repairInvalidJSONBackslashesWithPathContext(raw) - if repaired != raw { - if reparsed := decodeJSONObject(repaired); reparsed != nil { - return reparsed - } - } - - repairedLoose := RepairLooseJSON(raw) - if repairedLoose != raw { - if reparsed := decodeJSONObject(repairedLoose); reparsed != nil { - return reparsed - } - } - return map[string]any{"_raw": raw} - default: - b, err := json.Marshal(x) - if err != nil { - return map[string]any{} - } - var parsed map[string]any - if err := json.Unmarshal(b, &parsed); err == nil && parsed != nil { - return parsed - } - return map[string]any{} - } -} - -func decodeJSONObject(raw string) map[string]any { - var parsed map[string]any - if err := json.Unmarshal([]byte(raw), &parsed); err == nil && parsed != nil { - return parsed - } - return nil -} - -func hasSuspiciousPathControlChars(v any) bool { - switch x := v.(type) { - case map[string]any: - for key, value := range x { - if isPathLikeKey(key) && hasControlCharsInString(value) { - return true - } - if hasSuspiciousPathControlChars(value) { - return true - } - } - case []any: - for _, item := range x { - if hasSuspiciousPathControlChars(item) { - return true - } - } - } - return false -} - -func isPathLikeKey(key string) bool { - lower := strings.ToLower(strings.TrimSpace(key)) - if lower == "" { - return false - } - for _, candidate := range []string{"path", "file", "filepath", "filename", "cwd", "dir", "directory"} { - if lower == candidate || strings.HasSuffix(lower, "_"+candidate) || strings.HasSuffix(lower, candidate+"_path") { - return true - } - } - return false -} - -func hasControlCharsInString(v any) bool { - s, ok := v.(string) - if !ok { - return false - } - return strings.ContainsAny(s, "\n\r\t") -} diff --git a/internal/util/toolcalls_repair.go b/internal/util/toolcalls_repair.go deleted file mode 100644 index 185cb45..0000000 --- a/internal/util/toolcalls_repair.go +++ /dev/null @@ -1,276 +0,0 @@ -package util - -import ( - "regexp" - "strings" -) - -var unquotedKeyPattern = regexp.MustCompile(`([{,]\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*:`) - -// fallback pattern for shallow objects; scanner-based repair runs first. -var missingArrayBracketsPattern = regexp.MustCompile(`(:\s*)(\{(?:[^{}]|\{[^{}]*\})*\}(?:\s*,\s*\{(?:[^{}]|\{[^{}]*\})*\})+)`) - -func repairInvalidJSONBackslashes(s string) string { - return repairInvalidJSONBackslashesWithPathContext(s) -} - -func repairInvalidJSONBackslashesWithPathContext(s string) string { - if !strings.Contains(s, "\\") { - return s - } - var out strings.Builder - out.Grow(len(s) + 10) - - runes := []rune(s) - pathKeyContext := buildPathKeyStringMask(runes) - inString := false - escaped := false - stringStart := -1 - - for i := 0; i < len(runes); i++ { - r := runes[i] - if r == '"' && !escaped { - inString = !inString - if inString { - stringStart = i - } else { - stringStart = -1 - } - out.WriteRune(r) - escaped = false - continue - } - if r == '\\' && inString { - if i+1 < len(runes) { - next := runes[i+1] - if next == 'u' { - if i+5 < len(runes) && isHex4(runes[i+2:i+6]) { - out.WriteRune('\\') - out.WriteRune('u') - for _, hx := range runes[i+2 : i+6] { - out.WriteRune(hx) - } - i += 5 - escaped = false - continue - } - } else if shouldKeepEscape(next, pathKeyContext[stringStart]) { - out.WriteRune('\\') - out.WriteRune(next) - i++ - escaped = false - continue - } - } - out.WriteString("\\\\") - escaped = false - continue - } - out.WriteRune(r) - escaped = r == '\\' && !escaped - if r != '\\' { - escaped = false - } - } - return out.String() -} - -func shouldKeepEscape(next rune, inPathContext bool) bool { - switch next { - case '"', '\\', '/', 'b', 'f': - return true - case 'n', 'r', 't': - return !inPathContext - case 'u': - return true - default: - return false - } -} - -func buildPathKeyStringMask(runes []rune) map[int]bool { - mask := map[int]bool{} - inString := false - escaped := false - stringStart := -1 - var lastKey string - - for i := 0; i < len(runes); i++ { - r := runes[i] - if !inString { - if r == '"' { - inString = true - stringStart = i - } - continue - } - if escaped { - escaped = false - continue - } - if r == '\\' { - escaped = true - continue - } - if r != '"' { - continue - } - - value := string(runes[stringStart+1 : i]) - j := i + 1 - for j < len(runes) && (runes[j] == ' ' || runes[j] == '\n' || runes[j] == '\r' || runes[j] == '\t') { - j++ - } - if j < len(runes) && runes[j] == ':' { - lastKey = strings.ToLower(strings.TrimSpace(value)) - } else if isPathLikeKey(lastKey) { - mask[stringStart] = true - } - - inString = false - stringStart = -1 - } - return mask -} - -func RepairLooseJSON(s string) string { - s = strings.TrimSpace(s) - if s == "" { - return s - } - s = unquotedKeyPattern.ReplaceAllString(s, `$1"$2":`) - s = repairMissingArrayBracketsByScanner(s) - return missingArrayBracketsPattern.ReplaceAllString(s, `$1[$2]`) -} - -func repairMissingArrayBracketsByScanner(s string) string { - const maxScanLen = 200_000 - if len(s) == 0 || len(s) > maxScanLen { - return s - } - - var out strings.Builder - out.Grow(len(s) + 8) - i := 0 - for i < len(s) { - if s[i] != ':' { - out.WriteByte(s[i]) - i++ - continue - } - out.WriteByte(':') - i++ - for i < len(s) && isJSONWhitespace(s[i]) { - out.WriteByte(s[i]) - i++ - } - if i >= len(s) || s[i] != '{' { - continue - } - - start := i - end := scanJSONObjectEnd(s, start) - if end < 0 { - out.WriteString(s[start:]) - break - } - cursor := end - next := skipJSONWhitespace(s, cursor) - if next >= len(s) || s[next] != ',' { - out.WriteString(s[start:end]) - i = end - continue - } - - seqEnd := end - hasMultiple := false - for { - comma := skipJSONWhitespace(s, seqEnd) - if comma >= len(s) || s[comma] != ',' { - break - } - objStart := skipJSONWhitespace(s, comma+1) - if objStart >= len(s) || s[objStart] != '{' { - break - } - objEnd := scanJSONObjectEnd(s, objStart) - if objEnd < 0 { - break - } - hasMultiple = true - seqEnd = objEnd - } - if !hasMultiple { - out.WriteString(s[start:end]) - i = end - continue - } - - out.WriteByte('[') - out.WriteString(s[start:seqEnd]) - out.WriteByte(']') - i = seqEnd - } - return out.String() -} - -func scanJSONObjectEnd(s string, start int) int { - depth := 0 - inString := false - escaped := false - for i := start; i < len(s); i++ { - c := s[i] - if inString { - if escaped { - escaped = false - continue - } - if c == '\\' { - escaped = true - continue - } - if c == '"' { - inString = false - } - continue - } - if c == '"' { - inString = true - continue - } - if c == '{' { - depth++ - continue - } - if c == '}' { - depth-- - if depth == 0 { - return i + 1 - } - } - } - return -1 -} - -func skipJSONWhitespace(s string, i int) int { - for i < len(s) && isJSONWhitespace(s[i]) { - i++ - } - return i -} - -func isJSONWhitespace(b byte) bool { - return b == ' ' || b == '\n' || b == '\r' || b == '\t' -} - -func isHex4(seq []rune) bool { - if len(seq) != 4 { - return false - } - for _, r := range seq { - if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { - return false - } - } - return true -} diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index 10458df..2d29c1a 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -22,8 +22,8 @@ func TestParseToolCalls(t *testing.T) { func TestParseToolCallsFromFencedJSON(t *testing.T) { text := "I will call tools now\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"news\"}}]}\n```" calls := ParseToolCalls(text, []string{"search"}) - if len(calls) != 0 { - t.Fatalf("expected fenced tool_call example to be ignored, got %#v", calls) + if len(calls) != 1 { + t.Fatalf("expected fenced tool_call payload to be parsed, got %#v", calls) } } @@ -99,10 +99,10 @@ func TestFormatOpenAIToolCalls(t *testing.T) { } } -func TestParseStandaloneToolCallsOnlyMatchesStandalonePayload(t *testing.T) { +func TestParseStandaloneToolCallsSupportsMixedProsePayload(t *testing.T) { mixed := `这里是示例：{"tool_calls":[{"name":"search","input":{"q":"go"}}]}` - if calls := ParseStandaloneToolCalls(mixed, []string{"search"}); len(calls) != 0 { - t.Fatalf("expected standalone parser to ignore mixed prose, got %#v", calls) + if calls := ParseStandaloneToolCalls(mixed, []string{"search"}); len(calls) != 1 { + t.Fatalf("expected standalone parser to parse mixed prose payload, got %#v", calls) } standalone := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}` @@ -112,10 +112,10 @@ func TestParseStandaloneToolCallsOnlyMatchesStandalonePayload(t *testing.T) { } } -func TestParseStandaloneToolCallsIgnoresFencedCodeBlock(t *testing.T) { +func TestParseStandaloneToolCallsParsesFencedCodeBlock(t *testing.T) { fenced := "```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```" - if calls := ParseStandaloneToolCalls(fenced, []string{"search"}); len(calls) != 0 { - t.Fatalf("expected fenced tool_call example to be ignored, got %#v", calls) + if calls := ParseStandaloneToolCalls(fenced, []string{"search"}); len(calls) != 1 { + t.Fatalf("expected fenced tool_call payload to be parsed, got %#v", calls) } } @@ -288,7 +288,7 @@ func TestRepairInvalidJSONBackslashes(t *testing.T) { input string expected string }{ - {`{"path": "C:\Users\name"}`, `{"path": "C:\\Users\\name"}`}, + {`{"path": "C:\Users\name"}`, `{"path": "C:\\Users\name"}`}, {`{"cmd": "cd D:\git_codes"}`, `{"cmd": "cd D:\\git_codes"}`}, {`{"text": "line1\nline2"}`, `{"text": "line1\nline2"}`}, {`{"path": "D:\\back\\slash"}`, `{"path": "D:\\back\\slash"}`}, @@ -419,29 +419,9 @@ func TestParseToolCallsWithMixedWindowsPaths(t *testing.T) { } } -func TestParseToolCallsWithPathEscapesAndTextNewlines(t *testing.T) { - text := `{"name":"write_file","input":"{\"content\":\"line1\\nline2\",\"path\":\"D:\\tmp\\a.txt\"}"}` - availableTools := []string{"write_file"} - parsed := ParseToolCalls(text, availableTools) - if len(parsed) != 1 { - t.Fatalf("expected 1 parsed tool call, got %d", len(parsed)) - } - - content, _ := parsed[0].Input["content"].(string) - path, _ := parsed[0].Input["path"].(string) - if !strings.Contains(content, "line1\nline2") { - t.Fatalf("expected content to preserve newline semantics, got %q", content) - } - if strings.ContainsAny(path, "\n\r\t") { - t.Fatalf("expected path to avoid control chars, got %q", path) - } - if !strings.Contains(path, `D:\tmp\a.txt`) { - t.Fatalf("expected path with literal backslashes, got %q", path) - } -} - func TestRepairLooseJSONWithNestedObjects(t *testing.T) { - // 覆盖深层嵌套对象的方括号修复，避免 regex 单层能力带来的漂移。 + // 测试嵌套对象的修复：DeepSeek 幻觉输出，每个元素内部包含嵌套 {} + // 注意：正则只支持单层嵌套，不支持更深层次的嵌套 tests := []struct { name string input string @@ -507,11 +487,6 @@ func TestRepairLooseJSONWithNestedObjects(t *testing.T) { input: `"tasks": {"id":1}, {"id":2}, {"id":3}, {"id":4}, {"id":5}`, expected: `"tasks": [{"id":1}, {"id":2}, {"id":3}, {"id":4}, {"id":5}]`, }, - { - name: "深层嵌套对象", - input: `"todos": {"meta":{"a":{"b":1}},"content":"x"}, {"meta":{"a":{"b":2}},"content":"y"}`, - expected: `"todos": [{"meta":{"a":{"b":1}},"content":"x"}, {"meta":{"a":{"b":2}},"content":"y"}]`, - }, } for _, tt := range tests { diff --git a/internal/util/util_edge_test.go b/internal/util/util_edge_test.go index 876cd04..81d607e 100644 --- a/internal/util/util_edge_test.go +++ b/internal/util/util_edge_test.go @@ -409,8 +409,8 @@ func TestParseToolCallsWithFunctionWrapper(t *testing.T) { func TestParseStandaloneToolCallsFencedCodeBlock(t *testing.T) { fenced := "Here's an example:\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```\nDon't execute this." calls := ParseStandaloneToolCalls(fenced, []string{"search"}) - if len(calls) != 0 { - t.Fatalf("expected fenced code block ignored, got %d calls", len(calls)) + if len(calls) != 1 { + t.Fatalf("expected fenced code block to be parsed, got %d calls", len(calls)) } } diff --git a/tests/compat/expected/toolcalls_fenced_json.json b/tests/compat/expected/toolcalls_fenced_json.json index d740e67..124de59 100644 --- a/tests/compat/expected/toolcalls_fenced_json.json +++ b/tests/compat/expected/toolcalls_fenced_json.json @@ -1,6 +1,13 @@ { - "calls": [], - "sawToolCallSyntax": false, + "calls": [ + { + "name": "read_file", + "input": { + "path": "README.MD" + } + } + ], + "sawToolCallSyntax": true, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/compat/expected/toolcalls_standalone_fenced_example.json b/tests/compat/expected/toolcalls_standalone_fenced_example.json index d740e67..124de59 100644 --- a/tests/compat/expected/toolcalls_standalone_fenced_example.json +++ b/tests/compat/expected/toolcalls_standalone_fenced_example.json @@ -1,6 +1,13 @@ { - "calls": [], - "sawToolCallSyntax": false, + "calls": [ + { + "name": "read_file", + "input": { + "path": "README.MD" + } + } + ], + "sawToolCallSyntax": true, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/compat/expected/toolcalls_standalone_mixed_prose.json b/tests/compat/expected/toolcalls_standalone_mixed_prose.json index 0fcce27..124de59 100644 --- a/tests/compat/expected/toolcalls_standalone_mixed_prose.json +++ b/tests/compat/expected/toolcalls_standalone_mixed_prose.json @@ -1,6 +1,13 @@ { - "calls": [], + "calls": [ + { + "name": "read_file", + "input": { + "path": "README.MD" + } + } + ], "sawToolCallSyntax": true, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index 7c703a1..d4b5481 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -91,7 +91,9 @@ test('parseToolCalls supports fenced json and function.arguments string payload' '```', ].join('\n'); const calls = parseToolCalls(text, ['read_file']); - assert.equal(calls.length, 0); + assert.equal(calls.length, 1); + assert.equal(calls[0].name, 'read_file'); + assert.equal(calls[0].input.path, 'README.md'); }); test('parseToolCalls parses text-kv fallback payload', () => { @@ -122,19 +124,19 @@ test('parseToolCalls parses multiple text-kv fallback payloads', () => { assert.equal(calls[1].name, 'bash'); }); -test('parseStandaloneToolCalls only matches standalone payload and ignores mixed prose', () => { +test('parseStandaloneToolCalls parses mixed prose payload', () => { const mixed = '这里是示例：{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}，请勿执行。'; const standalone = '{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}'; const mixedCalls = parseStandaloneToolCalls(mixed, ['read_file']); const standaloneCalls = parseStandaloneToolCalls(standalone, ['read_file']); - assert.equal(mixedCalls.length, 0); + assert.equal(mixedCalls.length, 1); assert.equal(standaloneCalls.length, 1); }); -test('parseStandaloneToolCalls ignores fenced code block tool_call examples', () => { +test('parseStandaloneToolCalls parses fenced code block tool_call payload', () => { const fenced = ['```json', '{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}', '```'].join('\n'); const calls = parseStandaloneToolCalls(fenced, ['read_file']); - assert.equal(calls.length, 0); + assert.equal(calls.length, 1); });