diff --git a/internal/adapter/claude/handler_stream_test.go b/internal/adapter/claude/handler_stream_test.go index f5f7d75..aabc2f3 100644 --- a/internal/adapter/claude/handler_stream_test.go +++ b/internal/adapter/claude/handler_stream_test.go @@ -247,16 +247,18 @@ func asString(v any) string { func TestHandleClaudeStreamRealtimeToolSafetyAcrossStructuredFormats(t *testing.T) { tests := []struct { - name string - payload string + name string + payload string + wantToolUse bool }{ - {name: "xml_tool_call", payload: `Bashpwd`}, - {name: "xml_json_tool_call", payload: `{"tool":"Bash","params":{"command":"pwd"}}`}, - {name: "nested_tool_tag_style", payload: `pwd`}, - {name: "function_tag_style", payload: `Bashpwd`}, - {name: "antml_argument_style", payload: `pwd`}, - {name: "antml_function_attr_parameters", payload: `{"command":"pwd"}`}, - {name: "invoke_parameter_style", payload: `pwd`}, + {name: "canonical_tools_wrapper", payload: `Bashpwd`, wantToolUse: true}, + {name: "legacy_single_tool_root", payload: `Bashpwd`, wantToolUse: false}, + {name: "legacy_tool_call_json", payload: `{"tool":"Bash","params":{"command":"pwd"}}`, wantToolUse: false}, + {name: "legacy_nested_tool_tag_style", payload: `pwd`, wantToolUse: false}, + {name: "legacy_function_tag_style", payload: `Bashpwd`, wantToolUse: false}, + {name: "legacy_antml_argument_style", payload: `pwd`, wantToolUse: false}, + {name: "legacy_antml_function_attr_parameters", payload: `{"command":"pwd"}`, wantToolUse: false}, + {name: "legacy_invoke_parameter_style", payload: `pwd`, wantToolUse: false}, } for _, tc := range tests { @@ -280,8 +282,8 @@ func TestHandleClaudeStreamRealtimeToolSafetyAcrossStructuredFormats(t *testing. break } } - if !foundToolUse { - t.Fatalf("expected tool_use block for format %s, body=%s", tc.name, rec.Body.String()) + if foundToolUse != tc.wantToolUse { + t.Fatalf("unexpected tool_use=%v for format %s, body=%s", foundToolUse, tc.name, rec.Body.String()) } }) } @@ -289,7 +291,7 @@ func TestHandleClaudeStreamRealtimeToolSafetyAcrossStructuredFormats(t *testing. func TestHandleClaudeStreamRealtimeDetectsToolUseWithLeadingProse(t *testing.T) { h := &Handler{} - payload := "I'll call a tool now.\\nwrite_file{\\\"path\\\":\\\"/tmp/a.txt\\\",\\\"content\\\":\\\"abc\\\"}" + payload := "I'll call a tool now.\\nwrite_file{\\\"path\\\":\\\"/tmp/a.txt\\\",\\\"content\\\":\\\"abc\\\"}" resp := makeClaudeSSEHTTPResponse( `data: {"p":"response/content","v":"`+payload+`"}`, `data: [DONE]`, diff --git a/internal/adapter/claude/handler_util_test.go b/internal/adapter/claude/handler_util_test.go index 171c52a..7efa8dd 100644 --- a/internal/adapter/claude/handler_util_test.go +++ b/internal/adapter/claude/handler_util_test.go @@ -93,10 +93,10 @@ func TestNormalizeClaudeMessagesToolUseToAssistantToolCalls(t *testing.T) { t.Fatalf("expected call id preserved, got %#v", call) } content, _ := m["content"].(string) - if !containsStr(content, "") || !containsStr(content, "search_web") { + if !containsStr(content, "") || !containsStr(content, "search_web") { t.Fatalf("expected assistant content to include XML tool call history, got %q", content) } - if !containsStr(content, "\n \n ") { + if !containsStr(content, "\n \n ") { t.Fatalf("expected assistant content to include serialized parameters, got %q", content) } } @@ -292,7 +292,7 @@ func TestBuildClaudeToolPromptSingleTool(t *testing.T) { if !containsStr(prompt, "Search the web") { t.Fatalf("expected description in prompt") } - if !containsStr(prompt, "") { + if !containsStr(prompt, "") { t.Fatalf("expected XML tool_calls format in prompt") } if !containsStr(prompt, "TOOL CALL FORMAT") { diff --git a/internal/adapter/openai/chat_stream_runtime.go b/internal/adapter/openai/chat_stream_runtime.go index 1d7fff6..25124c7 100644 --- a/internal/adapter/openai/chat_stream_runtime.go +++ b/internal/adapter/openai/chat_stream_runtime.go @@ -201,17 +201,7 @@ func (s *chatStreamRuntime) finalize(finishReason string) { finishReason = "tool_calls" } if len(detected.Calls) == 0 && !s.toolCallsEmitted && strings.TrimSpace(finalText) == "" { - status := http.StatusTooManyRequests - message := "Upstream model returned empty output." - code := "upstream_empty_output" - if strings.TrimSpace(finalThinking) != "" { - message = "Upstream model returned reasoning without visible output." - } - if finishReason == "content_filter" { - status = http.StatusBadRequest - message = "Upstream content filtered the response and returned no output." - code = "content_filter" - } + status, message, code := upstreamEmptyOutputDetail(finishReason == "content_filter", finalText, finalThinking) s.sendFailedChunk(status, message, code) return } diff --git a/internal/adapter/openai/handler_chat.go b/internal/adapter/openai/handler_chat.go index 29636fd..4890e83 100644 --- a/internal/adapter/openai/handler_chat.go +++ b/internal/adapter/openai/handler_chat.go @@ -166,7 +166,7 @@ func (h *Handler) handleNonStream(w http.ResponseWriter, resp *http.Response, co if historySession != nil { historySession.error(status, message, code, finalThinking, finalText) } - writeUpstreamEmptyOutputError(w, finalText, result.ContentFilter) + writeUpstreamEmptyOutputError(w, finalText, finalThinking, result.ContentFilter) return } respBody := openaifmt.BuildChatCompletion(completionID, model, finalPrompt, finalThinking, finalText, toolNames) diff --git a/internal/adapter/openai/handler_toolcall_test.go b/internal/adapter/openai/handler_toolcall_test.go index 3a2e4e2..e2015fa 100644 --- a/internal/adapter/openai/handler_toolcall_test.go +++ b/internal/adapter/openai/handler_toolcall_test.go @@ -217,8 +217,8 @@ func TestHandleStreamIncompleteCapturedToolJSONFlushesAsTextOnFinalize(t *testin func TestHandleStreamEmitsDistinctToolCallIDsAcrossSeparateToolBlocks(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( - `data: {"p":"response/content","v":"前置文本\n\n \n read_file\n {\"path\":\"README.MD\"}\n \n"}`, - `data: {"p":"response/content","v":"中间文本\n\n \n search\n {\"q\":\"golang\"}\n \n"}`, + `data: {"p":"response/content","v":"前置文本\n\n \n read_file\n {\"path\":\"README.MD\"}\n \n"}`, + `data: {"p":"response/content","v":"中间文本\n\n \n search\n {\"q\":\"golang\"}\n \n"}`, `data: [DONE]`, ) rec := httptest.NewRecorder() diff --git a/internal/adapter/openai/history_split_test.go b/internal/adapter/openai/history_split_test.go index 1ba6777..554346b 100644 --- a/internal/adapter/openai/history_split_test.go +++ b/internal/adapter/openai/history_split_test.go @@ -76,7 +76,7 @@ func TestBuildOpenAIHistoryTranscriptUsesInjectedFileWrapper(t *testing.T) { if !strings.Contains(transcript, "[reasoning_content]") || !strings.Contains(transcript, "hidden reasoning") { t.Fatalf("expected reasoning block preserved, got %q", transcript) } - if !strings.Contains(transcript, "") { + if !strings.Contains(transcript, "") { t.Fatalf("expected tool calls preserved, got %q", transcript) } if !strings.HasSuffix(transcript, "\n[file name]: IGNORE\n[file content begin]\n") { diff --git a/internal/adapter/openai/message_normalize_test.go b/internal/adapter/openai/message_normalize_test.go index 564fea7..b7a4bb6 100644 --- a/internal/adapter/openai/message_normalize_test.go +++ b/internal/adapter/openai/message_normalize_test.go @@ -38,7 +38,7 @@ func TestNormalizeOpenAIMessagesForPrompt_AssistantToolCallsAndToolResult(t *tes t.Fatalf("expected 4 normalized messages with assistant tool history preserved, got %d", len(normalized)) } assistantContent, _ := normalized[2]["content"].(string) - if !strings.Contains(assistantContent, "") { + if !strings.Contains(assistantContent, "") { t.Fatalf("assistant tool history should be preserved in XML form, got %q", assistantContent) } if !strings.Contains(assistantContent, "get_weather") { @@ -49,7 +49,7 @@ func TestNormalizeOpenAIMessagesForPrompt_AssistantToolCallsAndToolResult(t *tes } prompt := util.MessagesPrepare(normalized) - if !strings.Contains(prompt, "") { + if !strings.Contains(prompt, "") { t.Fatalf("expected preserved assistant tool history in prompt: %q", prompt) } } @@ -258,7 +258,7 @@ func TestNormalizeOpenAIMessagesForPrompt_AssistantNilContentDoesNotInjectNullLi if strings.Contains(content, "null") { t.Fatalf("expected no null literal injection, got %q", content) } - if !strings.Contains(content, "") { + if !strings.Contains(content, "") { t.Fatalf("expected assistant tool history in normalized content, got %q", content) } } diff --git a/internal/adapter/openai/prompt_build_test.go b/internal/adapter/openai/prompt_build_test.go index 0d7e1c5..989399c 100644 --- a/internal/adapter/openai/prompt_build_test.go +++ b/internal/adapter/openai/prompt_build_test.go @@ -47,7 +47,7 @@ func TestBuildOpenAIFinalPrompt_HandlerPathIncludesToolRoundtripSemantics(t *tes if !strings.Contains(finalPrompt, `"condition":"sunny"`) { t.Fatalf("handler finalPrompt should preserve tool output content: %q", finalPrompt) } - if !strings.Contains(finalPrompt, "") { + if !strings.Contains(finalPrompt, "") { t.Fatalf("handler finalPrompt should preserve assistant tool history: %q", finalPrompt) } if !strings.Contains(finalPrompt, "get_weather") { @@ -74,7 +74,7 @@ func TestBuildOpenAIFinalPrompt_VercelPreparePathKeepsFinalAnswerInstruction(t * } finalPrompt, _ := buildOpenAIFinalPrompt(messages, tools, "", false) - if !strings.Contains(finalPrompt, "Remember: The ONLY valid way to use tools is the XML block at the end of your response.") { + if !strings.Contains(finalPrompt, "Remember: The ONLY valid way to use tools is the ... XML block at the end of your response.") { t.Fatalf("vercel prepare finalPrompt missing final tool-call anchor instruction: %q", finalPrompt) } if !strings.Contains(finalPrompt, "TOOL CALL FORMAT") { diff --git a/internal/adapter/openai/responses_handler.go b/internal/adapter/openai/responses_handler.go index 7d5be12..3518722 100644 --- a/internal/adapter/openai/responses_handler.go +++ b/internal/adapter/openai/responses_handler.go @@ -135,7 +135,7 @@ func (h *Handler) handleResponsesNonStream(w http.ResponseWriter, resp *http.Res if searchEnabled { sanitizedText = replaceCitationMarkersWithLinks(sanitizedText, result.CitationLinks) } - if writeUpstreamEmptyOutputError(w, sanitizedText, result.ContentFilter) { + if writeUpstreamEmptyOutputError(w, sanitizedText, sanitizedThinking, result.ContentFilter) { return } textParsed := toolcall.ParseStandaloneToolCallsDetailed(sanitizedText, toolNames) diff --git a/internal/adapter/openai/responses_stream_runtime_core.go b/internal/adapter/openai/responses_stream_runtime_core.go index af7eb8e..bba0b43 100644 --- a/internal/adapter/openai/responses_stream_runtime_core.go +++ b/internal/adapter/openai/responses_stream_runtime_core.go @@ -99,7 +99,7 @@ func newResponsesStreamRuntime( } } -func (s *responsesStreamRuntime) failResponse(message, code string) { +func (s *responsesStreamRuntime) failResponse(status int, message, code string) { s.failed = true failedResp := map[string]any{ "id": s.responseID, @@ -107,11 +107,12 @@ func (s *responsesStreamRuntime) failResponse(message, code string) { "object": "response", "model": s.model, "status": "failed", + "status_code": status, "output": []any{}, "output_text": "", "error": map[string]any{ "message": message, - "type": "invalid_request_error", + "type": openAIErrorType(status), "code": code, "param": nil, }, @@ -119,7 +120,7 @@ func (s *responsesStreamRuntime) failResponse(message, code string) { if s.persistResponse != nil { s.persistResponse(failedResp) } - s.sendEvent("response.failed", openaifmt.BuildResponsesFailedPayload(s.responseID, s.model, message, code)) + s.sendEvent("response.failed", openaifmt.BuildResponsesFailedPayload(s.responseID, s.model, status, message, code)) s.sendDone() } @@ -145,16 +146,12 @@ func (s *responsesStreamRuntime) finalize() { s.closeMessageItem() if s.toolChoice.IsRequired() && len(detected) == 0 { - s.failResponse("tool_choice requires at least one valid tool call.", "tool_choice_violation") + s.failResponse(http.StatusUnprocessableEntity, "tool_choice requires at least one valid tool call.", "tool_choice_violation") return } if len(detected) == 0 && strings.TrimSpace(finalText) == "" { - code := "upstream_empty_output" - message := "Upstream model returned empty output." - if finalThinking != "" { - message = "Upstream model returned reasoning without visible output." - } - s.failResponse(message, code) + status, message, code := upstreamEmptyOutputDetail(false, finalText, finalThinking) + s.failResponse(status, message, code) return } s.closeIncompleteFunctionItems() diff --git a/internal/adapter/openai/responses_stream_test.go b/internal/adapter/openai/responses_stream_test.go index 7999fa7..2fa2184 100644 --- a/internal/adapter/openai/responses_stream_test.go +++ b/internal/adapter/openai/responses_stream_test.go @@ -122,8 +122,8 @@ func TestHandleResponsesStreamEmitsDistinctToolCallIDsAcrossSeparateToolBlocks(t return "data: " + string(b) + "\n" } - streamBody := sseLine("前置文本\n\n \n read_file\n {\"path\":\"README.MD\"}\n \n") + - sseLine("中间文本\n\n \n search\n {\"q\":\"golang\"}\n \n") + + streamBody := sseLine("前置文本\n\n \n read_file\n {\"path\":\"README.MD\"}\n \n") + + sseLine("中间文本\n\n \n search\n {\"q\":\"golang\"}\n \n") + "data: [DONE]\n" resp := &http.Response{ StatusCode: http.StatusOK, diff --git a/internal/adapter/openai/tool_sieve_xml.go b/internal/adapter/openai/tool_sieve_xml.go index b019b93..d00c86b 100644 --- a/internal/adapter/openai/tool_sieve_xml.go +++ b/internal/adapter/openai/tool_sieve_xml.go @@ -9,22 +9,18 @@ import ( // --- XML tool call support for the streaming sieve --- //nolint:unused // kept as explicit tag inventory for future XML sieve refinements. -var xmlToolCallClosingTags = []string{"", "", "", "", "", "", +var xmlToolCallClosingTags = []string{"", "", // Agent-style XML tags (Roo Code, Cline, etc.) "", "", "", ""} -var xmlToolCallOpeningTags = []string{""}, + {""}, {""}, - {""}, - {""}, - {""}, - {""}, // Agent-style: these are XML "tool call" patterns from coding agents. // They get captured → parsed. If parsing fails, the raw XML is preserved // so the caller can still see the original text. @@ -36,11 +32,10 @@ var xmlToolCallTagPairs = []struct{ open, close string }{ // xmlToolCallBlockPattern matches a complete XML tool call block (wrapper or standalone). // //nolint:unused // reserved for future fast-path XML block detection. -var xmlToolCallBlockPattern = regexp.MustCompile(`(?is)(\s*(?:.*?)\s*|\s*(?:.*?)\s*|]*>(?:.*?)|]*>(?:.*?)|(?:.*?)|(?:.*?)|(?:.*?)|(?:.*?))`) +var xmlToolCallBlockPattern = regexp.MustCompile(`(?is)(]*>\s*(?:.*?)\s*|]*>(?:.*?)|(?:.*?)|(?:.*?)|(?:.*?))`) // xmlToolTagsToDetect is the set of XML tag prefixes used by findToolSegmentStart. -var xmlToolTagsToDetect = []string{"", "", "", "", +var xmlToolTagsToDetect = []string{"", "", "", "", ""} diff --git a/internal/adapter/openai/tool_sieve_xml_test.go b/internal/adapter/openai/tool_sieve_xml_test.go index 16827cc..9bd3acd 100644 --- a/internal/adapter/openai/tool_sieve_xml_test.go +++ b/internal/adapter/openai/tool_sieve_xml_test.go @@ -9,12 +9,12 @@ func TestProcessToolSieveInterceptsXMLToolCallWithoutLeak(t *testing.T) { var state toolStreamSieveState // Simulate a model producing XML tool call output chunk by chunk. chunks := []string{ - "\n", + "\n", " \n", " read_file\n", - ` {"path":"README.MD"}` + "\n", + ` {"path":"README.MD"}` + "\n", " \n", - "", + "", } var events []toolStreamEvent for _, c := range chunks { @@ -48,10 +48,10 @@ func TestProcessToolSieveHandlesLongXMLToolCall(t *testing.T) { payload := strings.Repeat("x", 4096) splitAt := len(payload) / 2 chunks := []string{ - "\n \n " + toolName + "\n \n \n \n " + toolName + "\n \n \n \n \n", + "]]>\n \n \n", } var events []toolStreamEvent @@ -90,8 +90,8 @@ func TestProcessToolSieveXMLWithLeadingText(t *testing.T) { // Model outputs some prose then an XML tool call. chunks := []string{ "Let me check the file.\n", - "\n \n read_file\n", - ` {"path":"go.mod"}` + "\n \n", + "\n \n read_file\n", + ` {"path":"go.mod"}` + "\n \n", } var events []toolStreamEvent for _, c := range chunks { @@ -123,7 +123,7 @@ func TestProcessToolSieveXMLWithLeadingText(t *testing.T) { func TestProcessToolSievePassesThroughNonToolXMLBlock(t *testing.T) { var state toolStreamSieveState - chunk := `示例 XMLplain text xml payload` + chunk := `示例 XMLplain text xml payload` events := processToolSieveChunk(&state, chunk, []string{"read_file"}) events = append(events, flushToolSieve(&state, []string{"read_file"})...) @@ -143,7 +143,7 @@ func TestProcessToolSievePassesThroughNonToolXMLBlock(t *testing.T) { func TestProcessToolSieveNonToolXMLKeepsSuffixForToolParsing(t *testing.T) { var state toolStreamSieveState - chunk := `plain xml{"path":"README.MD"}` + chunk := `plain xmlread_file{"path":"README.MD"}` events := processToolSieveChunk(&state, chunk, []string{"read_file"}) events = append(events, flushToolSieve(&state, []string{"read_file"})...) @@ -153,11 +153,11 @@ func TestProcessToolSieveNonToolXMLKeepsSuffixForToolParsing(t *testing.T) { textContent.WriteString(evt.Content) toolCalls += len(evt.ToolCalls) } - if !strings.Contains(textContent.String(), `plain xml`) { + if !strings.Contains(textContent.String(), `plain xml`) { t.Fatalf("expected leading non-tool XML to be preserved, got %q", textContent.String()) } - if strings.Contains(textContent.String(), ``) { - t.Fatalf("expected invoke tool XML to be intercepted, got %q", textContent.String()) + if strings.Contains(textContent.String(), ``) { + t.Fatalf("expected canonical tool XML to be intercepted, got %q", textContent.String()) } if toolCalls != 1 { t.Fatalf("expected exactly one parsed tool call from suffix, got %d events=%#v", toolCalls, events) @@ -166,7 +166,7 @@ func TestProcessToolSieveNonToolXMLKeepsSuffixForToolParsing(t *testing.T) { func TestProcessToolSievePassesThroughMalformedExecutableXMLBlock(t *testing.T) { var state toolStreamSieveState - chunk := `{"path":"README.md"}` + chunk := `{"path":"README.md"}` events := processToolSieveChunk(&state, chunk, []string{"read_file"}) events = append(events, flushToolSieve(&state, []string{"read_file"})...) @@ -189,17 +189,17 @@ func TestProcessToolSievePassesThroughFencedXMLToolCallExamples(t *testing.T) { var state toolStreamSieveState input := strings.Join([]string{ "Before first example.\n```", - "xml\nread_file{\"path\":\"README.md\"}\n```\n", + "xml\nread_file{\"path\":\"README.md\"}\n```\n", "Between examples.\n```xml\n", - "search{\"q\":\"golang\"}\n", + "search{\"q\":\"golang\"}\n", "```\nAfter examples.", }, "") chunks := []string{ "Before first example.\n```", - "xml\nread_file{\"path\":\"README.md\"}\n```\n", + "xml\nread_file{\"path\":\"README.md\"}\n```\n", "Between examples.\n```xml\n", - "search{\"q\":\"golang\"}\n", + "search{\"q\":\"golang\"}\n", "```\nAfter examples.", } @@ -230,13 +230,13 @@ func TestProcessToolSieveKeepsPartialXMLTagInsideFencedExample(t *testing.T) { var state toolStreamSieveState input := strings.Join([]string{ "Example:\n```xml\nread_file{\"path\":\"README.md\"}\n```\n", + "ll>read_file{\"path\":\"README.md\"}\n```\n", "Done.", }, "") chunks := []string{ "Example:\n```xml\nread_file{\"path\":\"README.md\"}\n```\n", + "ll>read_file{\"path\":\"README.md\"}\n```\n", "Done.", } @@ -288,11 +288,9 @@ func TestFindToolSegmentStartDetectsXMLToolCalls(t *testing.T) { input string want int }{ - {"tool_calls_tag", "some text \n", 10}, + {"tools_tag", "some text \n", 10}, {"tool_call_tag", "prefix \n", 7}, - {"invoke_tag", "text body", 5}, - {"xml_inside_code_fence", "```xml\nread_file\n```", -1}, - {"function_call_tag", "body", 0}, + {"xml_inside_code_fence", "```xml\nread_file\n```", -1}, {"no_xml", "just plain text", -1}, {"gemini_json_no_detect", `some text {"functionCall":{"name":"search"}}`, -1}, } @@ -312,10 +310,10 @@ func TestFindPartialXMLToolTagStart(t *testing.T) { input string want int }{ + {"partial_tools", "Hello done", -1}, + {"complete_tag", "Text done", -1}, {"no_lt", "plain text", -1}, {"closed_lt", "a < b > c", -1}, } @@ -330,10 +328,10 @@ func TestFindPartialXMLToolTagStart(t *testing.T) { } func TestHasOpenXMLToolTag(t *testing.T) { - if !hasOpenXMLToolTag("\nfoo") { + if !hasOpenXMLToolTag("\n\nfoo") { t.Fatal("should detect open XML tool tag without closing tag") } - if hasOpenXMLToolTag("\nfoo") { + if hasOpenXMLToolTag("\n\nfoo\n") { t.Fatal("should return false when closing tag is present") } if hasOpenXMLToolTag("plain text without any XML") { @@ -342,14 +340,14 @@ func TestHasOpenXMLToolTag(t *testing.T) { } // Test the EXACT scenario the user reports: token-by-token streaming where -// tag arrives in small pieces. +// tag arrives in small pieces. func TestProcessToolSieveTokenByTokenXMLNoLeak(t *testing.T) { var state toolStreamSieveState // Simulate DeepSeek model generating tokens one at a time. chunks := []string{ "<", "tool", - "_calls", + "s", ">\n", " <", "tool", @@ -366,21 +364,20 @@ func TestProcessToolSieveTokenByTokenXMLNoLeak(t *testing.T) { "_name", ">\n", " <", - "parameters", + "param", ">", `{"path"`, `: "README.MD"`, `}`, "\n", " \n", "", } var events []toolStreamEvent @@ -401,7 +398,7 @@ func TestProcessToolSieveTokenByTokenXMLNoLeak(t *testing.T) { if strings.Contains(textContent, "") { + if strings.Contains(textContent, "tools>") { t.Fatalf("closing tag fragment leaked to text: %q", textContent) } if strings.Contains(textContent, "read_file") { @@ -417,7 +414,7 @@ func TestFlushToolSieveIncompleteXMLFallsBackToText(t *testing.T) { var state toolStreamSieveState // XML block starts but stream ends before completion. chunks := []string{ - "\n", + "\n", " \n", " read_file\n", } @@ -440,19 +437,19 @@ func TestFlushToolSieveIncompleteXMLFallsBackToText(t *testing.T) { } } -// Test that the opening tag "\n " is NOT emitted as text content. +// Test that the opening tag "\n " is NOT emitted as text content. func TestOpeningXMLTagNotLeakedAsContent(t *testing.T) { var state toolStreamSieveState // First chunk is the opening tag - should be held, not emitted. - evts1 := processToolSieveChunk(&state, "\n ", []string{"read_file"}) + evts1 := processToolSieveChunk(&state, "\n ", []string{"read_file"}) for _, evt := range evts1 { - if strings.Contains(evt.Content, "") { + if strings.Contains(evt.Content, "") { t.Fatalf("opening tag leaked on first chunk: %q", evt.Content) } } // Remaining content arrives. - evts2 := processToolSieveChunk(&state, "\n read_file\n {\"path\":\"README.MD\"}\n \n", []string{"read_file"}) + evts2 := processToolSieveChunk(&state, "\n read_file\n {\"path\":\"README.MD\"}\n \n", []string{"read_file"}) evts2 = append(evts2, flushToolSieve(&state, []string{"read_file"})...) var textContent string diff --git a/internal/adapter/openai/upstream_empty.go b/internal/adapter/openai/upstream_empty.go index bb2da1f..23e82e6 100644 --- a/internal/adapter/openai/upstream_empty.go +++ b/internal/adapter/openai/upstream_empty.go @@ -12,16 +12,16 @@ func upstreamEmptyOutputDetail(contentFilter bool, text, thinking string) (int, return http.StatusBadRequest, "Upstream content filtered the response and returned no output.", "content_filter" } if thinking != "" { - return http.StatusTooManyRequests, "Upstream model returned reasoning without visible output.", "upstream_empty_output" + return http.StatusTooManyRequests, "Upstream account hit a rate limit and returned reasoning without visible output.", "upstream_empty_output" } - return http.StatusTooManyRequests, "Upstream model returned empty output.", "upstream_empty_output" + return http.StatusTooManyRequests, "Upstream account hit a rate limit and returned empty output.", "upstream_empty_output" } -func writeUpstreamEmptyOutputError(w http.ResponseWriter, text string, contentFilter bool) bool { +func writeUpstreamEmptyOutputError(w http.ResponseWriter, text, thinking string, contentFilter bool) bool { if !shouldWriteUpstreamEmptyOutputError(text) { return false } - status, message, code := upstreamEmptyOutputDetail(contentFilter, text, "") + status, message, code := upstreamEmptyOutputDetail(contentFilter, text, thinking) writeOpenAIErrorWithCode(w, status, message, code) return true } diff --git a/internal/format/openai/render_stream_events.go b/internal/format/openai/render_stream_events.go index 6c1121a..33c7c09 100644 --- a/internal/format/openai/render_stream_events.go +++ b/internal/format/openai/render_stream_events.go @@ -117,7 +117,7 @@ func BuildResponsesFunctionCallArgumentsDonePayload(responseID, itemID string, o } } -func BuildResponsesFailedPayload(responseID, model, message, code string) map[string]any { +func BuildResponsesFailedPayload(responseID, model string, status int, message, code string) map[string]any { code = strings.TrimSpace(code) if code == "" { code = "api_error" @@ -129,15 +129,36 @@ func BuildResponsesFailedPayload(responseID, model, message, code string) map[st "object": "response", "model": model, "status": "failed", + "status_code": status, "error": map[string]any{ "message": message, - "type": "invalid_request_error", + "type": responsesErrorType(status), "code": code, "param": nil, }, } } +func responsesErrorType(status int) string { + switch status { + case 400, 404, 422: + return "invalid_request_error" + case 401: + return "authentication_error" + case 403: + return "permission_error" + case 429: + return "rate_limit_error" + case 503: + return "service_unavailable_error" + default: + if status >= 500 { + return "api_error" + } + return "invalid_request_error" + } +} + func BuildResponsesCompletedPayload(response map[string]any) map[string]any { responseID, _ := response["id"].(string) return map[string]any{ diff --git a/internal/js/helpers/stream-tool-sieve/parse.js b/internal/js/helpers/stream-tool-sieve/parse.js index f6bb865..b7993c9 100644 --- a/internal/js/helpers/stream-tool-sieve/parse.js +++ b/internal/js/helpers/stream-tool-sieve/parse.js @@ -8,7 +8,7 @@ const { stripFencedCodeBlocks, } = require('./parse_payload'); -const TOOL_MARKUP_PREFIXES = [']*)>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi; -const TOOL_CALL_MARKUP_SELFCLOSE_PATTERN = /<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)\/>/gi; +const TOOLS_WRAPPER_PATTERN = /]*>([\s\S]*?)<\/tools>/gi; +const TOOL_CALL_MARKUP_BLOCK_PATTERN = /<(?:[a-z0-9_:-]+:)?tool_call\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?tool_call>/gi; +const TOOL_CALL_CANONICAL_BODY_PATTERN = /^\s*<(?:[a-z0-9_:-]+:)?tool_name\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?tool_name>\s*<(?:[a-z0-9_:-]+:)?param\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?param>\s*$/i; const TOOL_CALL_MARKUP_KV_PATTERN = /<(?:[a-z0-9_:-]+:)?([a-z0-9_.-]+)\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?\1>/gi; -const TOOL_CALL_MARKUP_ATTR_PATTERN = /(name|function|tool)\s*=\s*"([^"]+)"/i; -const TOOL_CALL_MARKUP_NAME_PATTERNS = [ - /<(?:[a-z0-9_:-]+:)?tool_name\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?tool_name>/i, - /<(?:[a-z0-9_:-]+:)?function_name\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?function_name>/i, - /<(?:[a-z0-9_:-]+:)?name\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?name>/i, - /<(?:[a-z0-9_:-]+:)?function\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?function>/i, -]; -const TOOL_CALL_MARKUP_ARGS_PATTERNS = [ - /<(?:[a-z0-9_:-]+:)?input\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?input>/i, - /<(?:[a-z0-9_:-]+:)?arguments\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?arguments>/i, - /<(?:[a-z0-9_:-]+:)?argument\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?argument>/i, - /<(?:[a-z0-9_:-]+:)?parameters\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?parameters>/i, - /<(?:[a-z0-9_:-]+:)?parameter\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?parameter>/i, - /<(?:[a-z0-9_:-]+:)?args\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?args>/i, - /<(?:[a-z0-9_:-]+:)?params\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9_:-]+:)?params>/i, -]; const CDATA_PATTERN = /^$/i; -const HTML_ENTITIES_PATTERN = /&[a-z0-9#]+;/gi; const { toStringSafe, @@ -40,22 +24,19 @@ function parseMarkupToolCalls(text) { return []; } const out = []; - for (const m of raw.matchAll(TOOL_CALL_MARKUP_BLOCK_PATTERN)) { - const parsed = parseMarkupSingleToolCall(toStringSafe(m[2]).trim(), toStringSafe(m[3]).trim()); - if (parsed) { - out.push(parsed); - } - } - for (const m of raw.matchAll(TOOL_CALL_MARKUP_SELFCLOSE_PATTERN)) { - const parsed = parseMarkupSingleToolCall(toStringSafe(m[1]).trim(), ''); - if (parsed) { - out.push(parsed); + for (const wrapper of raw.matchAll(TOOLS_WRAPPER_PATTERN)) { + const body = toStringSafe(wrapper[1]); + for (const block of body.matchAll(TOOL_CALL_MARKUP_BLOCK_PATTERN)) { + const parsed = parseMarkupSingleToolCall(toStringSafe(block[1]).trim()); + if (parsed) { + out.push(parsed); + } } } return out; } -function parseMarkupSingleToolCall(attrs, inner) { +function parseMarkupSingleToolCall(inner) { // Try inline JSON parse for the inner content. if (inner) { try { @@ -70,28 +51,18 @@ function parseMarkupSingleToolCall(attrs, inner) { // Not JSON, continue with markup parsing. } } - let name = ''; - const attrMatch = attrs.match(TOOL_CALL_MARKUP_ATTR_PATTERN); - if (attrMatch && attrMatch[2]) { - name = toStringSafe(attrMatch[2]).trim(); - } - if (!name) { - name = extractRawTagValue(findMarkupTagValue(inner, TOOL_CALL_MARKUP_NAME_PATTERNS)); + + const match = inner.match(TOOL_CALL_CANONICAL_BODY_PATTERN); + if (!match || match.length < 3) { + return null; } + + const name = extractRawTagValue(match[1]).trim(); if (!name) { return null; } - let input = {}; - const argsRaw = findMarkupTagValue(inner, TOOL_CALL_MARKUP_ARGS_PATTERNS); - if (argsRaw) { - input = parseMarkupInput(argsRaw); - } else { - const kv = parseMarkupKVObject(inner); - if (Object.keys(kv).length > 0) { - input = kv; - } - } + const input = parseMarkupInput(match[2]); return { name, input }; } @@ -187,21 +158,6 @@ function unescapeHtml(safe) { .replace(/'/g, "'"); } -function stripTagText(text) { - return toStringSafe(text).replace(/<[^>]+>/g, ' ').trim(); -} - -function findMarkupTagValue(text, patterns) { - const source = toStringSafe(text); - for (const p of patterns) { - const m = source.match(p); - if (m && m[1] !== undefined) { - return toStringSafe(m[1]); - } - } - return ''; -} - function parseToolCallInput(v) { if (v == null) { return {}; diff --git a/internal/js/helpers/stream-tool-sieve/sieve-xml.js b/internal/js/helpers/stream-tool-sieve/sieve-xml.js index 6442dbc..6eb5280 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve-xml.js +++ b/internal/js/helpers/stream-tool-sieve/sieve-xml.js @@ -3,12 +3,8 @@ const { parseToolCalls } = require('./parse'); // Tag pairs ordered longest-first: wrapper tags checked before inner tags. const XML_TOOL_TAG_PAIRS = [ - { open: '' }, + { open: '' }, { open: '' }, - { open: '' }, - { open: '' }, - { open: '' }, - { open: '' }, ]; const XML_TOOL_OPENING_TAGS = XML_TOOL_TAG_PAIRS.map(p => p.open); diff --git a/internal/js/helpers/stream-tool-sieve/tool-keywords.js b/internal/js/helpers/stream-tool-sieve/tool-keywords.js index ed7fbe0..473600b 100644 --- a/internal/js/helpers/stream-tool-sieve/tool-keywords.js +++ b/internal/js/helpers/stream-tool-sieve/tool-keywords.js @@ -1,16 +1,15 @@ 'use strict'; const XML_TOOL_SEGMENT_TAGS = [ - '', '', '', '', + '', '', '', '', '', '', '', '', + '', '', ]; module.exports = { @@ -18,4 +17,3 @@ module.exports = { XML_TOOL_OPENING_TAGS, XML_TOOL_CLOSING_TAGS, }; - diff --git a/internal/prompt/tool_calls.go b/internal/prompt/tool_calls.go index 4c14f6b..b24234d 100644 --- a/internal/prompt/tool_calls.go +++ b/internal/prompt/tool_calls.go @@ -38,7 +38,7 @@ func FormatToolCallsForPrompt(raw any) string { if len(blocks) == 0 { return "" } - return "\n" + strings.Join(blocks, "\n") + "\n" + return "\n" + strings.Join(blocks, "\n") + "\n" } // StringifyToolCallArguments normalizes tool arguments into a compact string @@ -105,16 +105,16 @@ func formatToolCallParametersForPrompt(raw any) string { body, ok := renderPromptToolXMLBody(value, " ") if ok { if strings.TrimSpace(body) == "" { - return " " + return " " } - return " \n" + body + "\n " + return " \n" + body + "\n " } fallback := StringifyToolCallArguments(raw) if strings.TrimSpace(fallback) == "" { fallback = "{}" } - return " " + renderPromptXMLText(fallback) + "" + return " " + renderPromptXMLText(fallback) + "" } func normalizePromptToolCallValue(raw any) any { diff --git a/internal/prompt/tool_calls_test.go b/internal/prompt/tool_calls_test.go index 2d30770..451120b 100644 --- a/internal/prompt/tool_calls_test.go +++ b/internal/prompt/tool_calls_test.go @@ -22,7 +22,7 @@ func TestFormatToolCallsForPromptXML(t *testing.T) { if got == "" { t.Fatal("expected non-empty formatted tool calls") } - if got != "\n \n search_web\n \n \n \n \n" { + if got != "\n \n search_web\n \n \n \n \n" { t.Fatalf("unexpected formatted tool call XML: %q", got) } } @@ -34,7 +34,7 @@ func TestFormatToolCallsForPromptEscapesXMLEntities(t *testing.T) { "arguments": `{"q":"a < b && c > d"}`, }, }) - want := "\n \n search<&>\n \n d]]>\n \n \n" + want := "\n \n search<&>\n \n d]]>\n \n \n" if got != want { t.Fatalf("unexpected escaped tool call XML: %q", got) } @@ -50,7 +50,7 @@ func TestFormatToolCallsForPromptUsesCDATAForMultilineContent(t *testing.T) { }, }, }) - want := "\n \n write_file\n \n \n \n \n \n" + want := "\n \n write_file\n \n \n \n \n \n" if got != want { t.Fatalf("unexpected multiline cdata tool call XML: %q", got) } diff --git a/internal/toolcall/regression_test.go b/internal/toolcall/regression_test.go index d268374..0e58952 100644 --- a/internal/toolcall/regression_test.go +++ b/internal/toolcall/regression_test.go @@ -13,18 +13,18 @@ func TestRegression_RobustXMLAndCDATA(t *testing.T) { }{ { name: "Standard JSON parameters (Regression)", - text: `foo{"a": 1}`, + text: `foo{"a": 1}`, expected: []ParsedToolCall{{Name: "foo", Input: map[string]any{"a": float64(1)}}}, }, { name: "XML tags parameters (Regression)", - text: `foohello`, + text: `foohello`, expected: []ParsedToolCall{{Name: "foo", Input: map[string]any{"arg1": "hello"}}}, }, { name: "CDATA parameters (New Feature)", - text: `write_file and & symbols]]>`, + text: `write_file and & symbols]]>`, expected: []ParsedToolCall{{ Name: "write_file", Input: map[string]any{"content": "line 1\nline 2 with and & symbols"}, @@ -32,9 +32,9 @@ line 2 with and & symbols]]>`, }, { name: "Nested XML with repeated parameters (New Feature)", - text: `write_filescript.shwrite_filescript.shfirstsecond`, +]]>firstsecond`, expected: []ParsedToolCall{{ Name: "write_file", Input: map[string]any{ @@ -46,7 +46,7 @@ echo "hello" }, { name: "Dirty XML with unescaped symbols (Robustness Improvement)", - text: `bashecho "hello" > out.txt && cat out.txt`, + text: `bashecho "hello" > out.txt && cat out.txt`, expected: []ParsedToolCall{{ Name: "bash", Input: map[string]any{"command": "echo \"hello\" > out.txt && cat out.txt"}, @@ -54,7 +54,7 @@ echo "hello" }, { name: "Mixed JSON inside CDATA (New Hybrid Case)", - text: `foo`, + text: `foo`, expected: []ParsedToolCall{{ Name: "foo", Input: map[string]any{"json_param": "works"}, diff --git a/internal/toolcall/tool_prompt.go b/internal/toolcall/tool_prompt.go index 8e896e9..7990ef8 100644 --- a/internal/toolcall/tool_prompt.go +++ b/internal/toolcall/tool_prompt.go @@ -36,19 +36,19 @@ func BuildToolCallInstructions(toolNames []string) string { return `TOOL CALL FORMAT — FOLLOW EXACTLY: - + TOOL_NAME_HERE - + - + - + RULES: -1) Use the XML format only. Never emit JSON or function-call syntax. -2) Put one or more entries under a single root. -3) Parameters must be XML, not JSON. +1) Use the XML wrapper format only. +2) Put one or more entries under a single root. +3) Use for the tool name and for the argument container. 4) All string values must use , even short ones. This includes code, scripts, file contents, prompts, paths, names, and queries. 5) Objects use nested XML elements. Arrays may repeat the same tag or use children. 6) Numbers, booleans, and null stay plain text. @@ -64,53 +64,51 @@ PARAMETER SHAPES: 【WRONG — Do NOT do these】: Wrong 1 — mixed text after XML: - ... I hope this helps. -Wrong 2 — function-call syntax: - Grep({"pattern": "token"}) -Wrong 3 — JSON parameters: - ` + ex1 + `{"path":"x"} -Wrong 4 — Markdown code fences: + ... I hope this helps. +Wrong 2 — JSON payload inside : + ` + ex1 + `{"path":"x"} +Wrong 3 — Markdown code fences: ` + "```xml" + ` - ... + ... ` + "```" + ` -Remember: The ONLY valid way to use tools is the XML block at the end of your response. +Remember: The ONLY valid way to use tools is the ... XML block at the end of your response. 【CORRECT EXAMPLES】: Example A — Single tool: - + ` + ex1 + ` - ` + ex1Params + ` + ` + ex1Params + ` - + Example B — Two tools in parallel: - + ` + ex1 + ` - ` + ex1Params + ` + ` + ex1Params + ` ` + ex2 + ` - ` + ex2Params + ` + ` + ex2Params + ` - + Example C — Tool with nested XML parameters: - + ` + ex3 + ` - ` + ex3Params + ` + ` + ex3Params + ` - + Example D — Tool with long script using CDATA (RELIABLE FOR CODE/SCRIPTS): - + ` + ex2 + ` - + ` + promptCDATA("script.sh") + ` - + - + ` } diff --git a/internal/toolcall/tool_prompt_test.go b/internal/toolcall/tool_prompt_test.go index 67aeb27..85cdeb4 100644 --- a/internal/toolcall/tool_prompt_test.go +++ b/internal/toolcall/tool_prompt_test.go @@ -10,7 +10,7 @@ func TestBuildToolCallInstructions_ExecCommandUsesCmdExample(t *testing.T) { if !strings.Contains(out, `exec_command`) { t.Fatalf("expected exec_command in examples, got: %s", out) } - if !strings.Contains(out, ``) { + if !strings.Contains(out, ``) { t.Fatalf("expected cmd parameter example for exec_command, got: %s", out) } } @@ -20,7 +20,7 @@ func TestBuildToolCallInstructions_ExecuteCommandUsesCommandExample(t *testing.T if !strings.Contains(out, `execute_command`) { t.Fatalf("expected execute_command in examples, got: %s", out) } - if !strings.Contains(out, ``) { + if !strings.Contains(out, ``) { t.Fatalf("expected command parameter example for execute_command, got: %s", out) } } diff --git a/internal/toolcall/toolcalls_input_parse.go b/internal/toolcall/toolcalls_input_parse.go index b987e64..4b7ef7e 100644 --- a/internal/toolcall/toolcalls_input_parse.go +++ b/internal/toolcall/toolcalls_input_parse.go @@ -2,6 +2,7 @@ package toolcall import ( "encoding/json" + "html" "strings" "unicode" ) @@ -13,7 +14,7 @@ func parseToolCallInput(v any) map[string]any { case map[string]any: return x case string: - raw := strings.TrimSpace(x) + raw := strings.TrimSpace(html.UnescapeString(x)) if raw == "" { return map[string]any{} } diff --git a/internal/toolcall/toolcalls_markup.go b/internal/toolcall/toolcalls_markup.go index 94420dc..3d8e657 100644 --- a/internal/toolcall/toolcalls_markup.go +++ b/internal/toolcall/toolcalls_markup.go @@ -7,120 +7,10 @@ import ( "strings" ) -var toolCallMarkupTagNames = []string{"tool_call", "function_call", "invoke"} -var toolCallMarkupTagPatternByName = map[string]*regexp.Regexp{ - "tool_call": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?tool_call\b([^>]*)>(.*?)`), - "function_call": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function_call\b([^>]*)>(.*?)`), - "invoke": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)>(.*?)`), -} -var toolCallMarkupSelfClosingPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)/>`) var toolCallMarkupKVPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)\b[^>]*>(.*?)`) -var toolCallMarkupAttrPattern = regexp.MustCompile(`(?is)(name|function|tool)\s*=\s*"([^"]+)"`) -var anyTagPattern = regexp.MustCompile(`(?is)<[^>]+>`) -var toolCallMarkupNameTagNames = []string{"name", "function"} -var toolCallMarkupNamePatternByTag = map[string]*regexp.Regexp{ - "name": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?name\b[^>]*>(.*?)`), - "function": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function\b[^>]*>(.*?)`), -} // cdataPattern matches a standalone CDATA section. var cdataPattern = regexp.MustCompile(`(?is)^$`) -var toolCallMarkupArgsTagNames = []string{"input", "arguments", "argument", "parameters", "parameter", "args", "params"} -var toolCallMarkupArgsPatternByTag = map[string]*regexp.Regexp{ - "input": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?input\b[^>]*>(.*?)`), - "arguments": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?arguments\b[^>]*>(.*?)`), - "argument": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?argument\b[^>]*>(.*?)`), - "parameters": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?parameters\b[^>]*>(.*?)`), - "parameter": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?parameter\b[^>]*>(.*?)`), - "args": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?args\b[^>]*>(.*?)`), - "params": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?params\b[^>]*>(.*?)`), -} - -func parseMarkupToolCalls(text string) []ParsedToolCall { - trimmed := strings.TrimSpace(text) - if trimmed == "" { - return nil - } - - out := make([]ParsedToolCall, 0) - for _, tagName := range toolCallMarkupTagNames { - pattern := toolCallMarkupTagPatternByName[tagName] - for _, m := range pattern.FindAllStringSubmatch(trimmed, -1) { - if len(m) < 3 { - continue - } - attrs := strings.TrimSpace(m[1]) - inner := strings.TrimSpace(m[2]) - if parsed := parseMarkupSingleToolCall(attrs, inner); parsed.Name != "" { - out = append(out, parsed) - } - } - } - for _, m := range toolCallMarkupSelfClosingPattern.FindAllStringSubmatch(trimmed, -1) { - if len(m) < 2 { - continue - } - if parsed := parseMarkupSingleToolCall(strings.TrimSpace(m[1]), ""); parsed.Name != "" { - out = append(out, parsed) - } - } - if len(out) == 0 { - return nil - } - return out -} - -func parseMarkupSingleToolCall(attrs string, inner string) ParsedToolCall { - // Try parsing inner content as a JSON tool call object. - if raw := strings.TrimSpace(inner); raw != "" && strings.HasPrefix(raw, "{") { - var obj map[string]any - if err := json.Unmarshal([]byte(raw), &obj); err == nil { - name, _ := obj["name"].(string) - if name == "" { - if fn, ok := obj["function"].(map[string]any); ok { - name, _ = fn["name"].(string) - } - } - if name == "" { - if fc, ok := obj["functionCall"].(map[string]any); ok { - name, _ = fc["name"].(string) - } - } - if strings.TrimSpace(name) != "" { - input := parseToolCallInput(obj["input"]) - if len(input) == 0 { - if args, ok := obj["arguments"]; ok { - input = parseToolCallInput(args) - } - } - return ParsedToolCall{Name: strings.TrimSpace(name), Input: input} - } - } - } - - name := "" - if m := toolCallMarkupAttrPattern.FindStringSubmatch(attrs); len(m) >= 3 { - name = strings.TrimSpace(m[2]) - } - if name == "" { - name = findMarkupTagValue(inner, toolCallMarkupNameTagNames, toolCallMarkupNamePatternByTag) - } - if name == "" { - return ParsedToolCall{} - } - - input := map[string]any{} - if argsRaw := findMarkupTagValue(inner, toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); argsRaw != "" { - input = parseMarkupInput(argsRaw) - } else if kv := parseMarkupKVObject(inner); len(kv) > 0 { - input = kv - } - return ParsedToolCall{Name: name, Input: input} -} - -func parseMarkupInput(raw string) map[string]any { - return parseStructuredToolCallInput(raw) -} func parseMarkupKVObject(text string) map[string]any { matches := toolCallMarkupKVPattern.FindAllStringSubmatch(strings.TrimSpace(text), -1) @@ -212,23 +102,3 @@ func extractRawTagValue(inner string) string { // but for KV objects we usually want the value. return html.UnescapeString(inner) } - -func stripTagText(text string) string { - return strings.TrimSpace(anyTagPattern.ReplaceAllString(text, "")) -} - -func findMarkupTagValue(text string, tagNames []string, patternByTag map[string]*regexp.Regexp) string { - for _, tag := range tagNames { - pattern := patternByTag[tag] - if pattern == nil { - continue - } - if m := pattern.FindStringSubmatch(text); len(m) >= 2 { - value := extractRawTagValue(m[1]) - if value != "" { - return value - } - } - } - return "" -} diff --git a/internal/toolcall/toolcalls_parse.go b/internal/toolcall/toolcalls_parse.go index bc61124..70c1529 100644 --- a/internal/toolcall/toolcalls_parse.go +++ b/internal/toolcall/toolcalls_parse.go @@ -46,9 +46,6 @@ func parseToolCallsDetailedXMLOnly(text string) ToolCallParseResult { } parsed := parseXMLToolCalls(trimmed) - if len(parsed) == 0 { - parsed = parseMarkupToolCalls(trimmed) - } if len(parsed) == 0 { return result } @@ -77,12 +74,8 @@ func filterToolCallsDetailed(parsed []ParsedToolCall) ([]ParsedToolCall, []strin func looksLikeToolCallSyntax(text string) bool { lower := strings.ToLower(text) - return strings.Contains(lower, "\s*(.*?)\s*`) -var functionCallPattern = regexp.MustCompile(`(?is)\s*([^<]+?)\s*`) -var functionParamPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) -var antmlFunctionCallPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?function_call[^>]*(?:name|function)="([^"]+)"[^>]*>\s*(.*?)\s*`) -var antmlArgumentPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?argument\s+name="([^"]+)"\s*>\s*(.*?)\s*`) -var invokeCallPattern = regexp.MustCompile(`(?is)(.*?)`) -var invokeParamPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) -var toolUseFunctionPattern = regexp.MustCompile(`(?is)\s*(.*?)\s*`) -var toolUseNameParametersPattern = regexp.MustCompile(`(?is)\s*\s*([^<]+?)\s*\s*\s*(.*?)\s*\s*`) -var toolUseFunctionNameParametersPattern = regexp.MustCompile(`(?is)\s*\s*([^<]+?)\s*\s*\s*(.*?)\s*\s*`) -var toolUseToolNameBodyPattern = regexp.MustCompile(`(?is)\s*\s*([^<]+?)\s*\s*(.*?)\s*`) -var xmlToolNamePatterns = []*regexp.Regexp{ - regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?tool_name\b[^>]*>(.*?)`), - regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function_name\b[^>]*>(.*?)`), -} +var xmlToolsWrapperPattern = regexp.MustCompile(`(?is)]*>\s*(.*?)\s*`) +var xmlToolCallPattern = regexp.MustCompile(`(?is)]*>\s*(.*?)\s*`) +var xmlCanonicalToolCallBodyPattern = regexp.MustCompile(`(?is)^\s*<(?:[a-z0-9_:-]+:)?tool_name\b[^>]*>(.*?)\s*<(?:[a-z0-9_:-]+:)?param\b[^>]*>(.*?)\s*$`) func parseXMLToolCalls(text string) []ParsedToolCall { - matches := xmlToolCallPattern.FindAllString(text, -1) - out := make([]ParsedToolCall, 0, len(matches)+1) - for _, block := range matches { - call, ok := parseSingleXMLToolCall(block) - if !ok { + wrappers := xmlToolsWrapperPattern.FindAllStringSubmatch(text, -1) + if len(wrappers) == 0 { + return nil + } + out := make([]ParsedToolCall, 0, len(wrappers)) + for _, wrapper := range wrappers { + if len(wrapper) < 2 { continue } - out = append(out, call) + for _, block := range xmlToolCallPattern.FindAllString(wrapper[1], -1) { + call, ok := parseSingleXMLToolCall(block) + if !ok { + continue + } + out = append(out, call) + } } - if len(out) > 0 { - return out + if len(out) == 0 { + return nil } - if call, ok := parseFunctionCallTagStyle(text); ok { - return []ParsedToolCall{call} - } - if calls := parseAntmlFunctionCallStyles(text); len(calls) > 0 { - return calls - } - if call, ok := parseInvokeFunctionCallStyle(text); ok { - return []ParsedToolCall{call} - } - if call, ok := parseToolUseFunctionStyle(text); ok { - return []ParsedToolCall{call} - } - if call, ok := parseToolUseNameParametersStyle(text); ok { - return []ParsedToolCall{call} - } - if call, ok := parseToolUseFunctionNameParametersStyle(text); ok { - return []ParsedToolCall{call} - } - if call, ok := parseToolUseToolNameBodyStyle(text); ok { - return []ParsedToolCall{call} - } - return nil + return out } func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { @@ -69,15 +43,10 @@ func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { if strings.HasPrefix(inner, "{") { var payload map[string]any if err := json.Unmarshal([]byte(inner), &payload); err == nil { - name := strings.TrimSpace(asString(payload["tool"])) - if name == "" { - name = strings.TrimSpace(asString(payload["tool_name"])) - } + name := strings.TrimSpace(asString(payload["name"])) if name != "" { input := map[string]any{} - if params, ok := payload["params"].(map[string]any); ok { - input = params - } else if params, ok := payload["parameters"].(map[string]any); ok { + if params, ok := payload["input"].(map[string]any); ok { input = params } return ParsedToolCall{Name: name, Input: input}, true @@ -85,350 +54,15 @@ func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) { } } - name := "" - params := extractXMLToolParamsByRegex(inner) - dec := xml.NewDecoder(strings.NewReader(block)) - inTool := false - for { - tok, err := dec.Token() - if err != nil { - break - } - switch t := tok.(type) { - case xml.StartElement: - tag := strings.ToLower(t.Name.Local) - switch tag { - case "tool": - inTool = true - for _, attr := range t.Attr { - if strings.EqualFold(strings.TrimSpace(attr.Name.Local), "name") && strings.TrimSpace(name) == "" { - name = strings.TrimSpace(attr.Value) - } - } - case "parameters": - var node struct { - Inner string `xml:",innerxml"` - } - if err := dec.DecodeElement(&node, &t); err == nil { - inner := strings.TrimSpace(node.Inner) - if inner != "" { - extracted := extractRawTagValue(inner) - if parsed := parseStructuredToolCallInput(extracted); len(parsed) > 0 { - for k, vv := range parsed { - params[k] = vv - } - } - } - } - case "tool_name", "function_name", "name": - var v string - if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" { - name = strings.TrimSpace(v) - } - case "input", "arguments", "argument", "args", "params": - var v string - if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" { - if parsed := parseStructuredToolCallInput(strings.TrimSpace(v)); len(parsed) > 0 { - for k, vv := range parsed { - params[k] = vv - } - } - } - default: - if inTool { - var v string - if err := dec.DecodeElement(&v, &t); err == nil { - params[t.Name.Local] = strings.TrimSpace(html.UnescapeString(v)) - } - } - } - case xml.EndElement: - tag := strings.ToLower(t.Name.Local) - if tag == "tool" { - inTool = false - } - } - } - if strings.TrimSpace(name) == "" { - name = strings.TrimSpace(html.UnescapeString(extractXMLToolNameByRegex(stripTopLevelXMLParameters(inner)))) + m := xmlCanonicalToolCallBodyPattern.FindStringSubmatch(inner) + if len(m) < 3 { + return ParsedToolCall{}, false } + name := strings.TrimSpace(html.UnescapeString(extractRawTagValue(m[1]))) if strings.TrimSpace(name) == "" { return ParsedToolCall{}, false } - return ParsedToolCall{Name: strings.TrimSpace(html.UnescapeString(name)), Input: params}, true -} - -func stripTopLevelXMLParameters(inner string) string { - out := strings.TrimSpace(inner) - for { - idx := strings.Index(strings.ToLower(out), "") - if openEnd < 0 { - return out - } - closeIdx := strings.Index(segmentLower, "") - if closeIdx < 0 { - return out[:idx] - } - end := idx + closeIdx + len("") - out = out[:idx] + out[end:] - } -} - -func extractXMLToolNameByRegex(inner string) string { - for _, pattern := range xmlToolNamePatterns { - if m := pattern.FindStringSubmatch(inner); len(m) >= 2 { - if v := strings.TrimSpace(stripTagText(m[1])); v != "" { - return v - } - } - } - return "" -} - -func extractXMLToolParamsByRegex(inner string) map[string]any { - raw := findMarkupTagValue(inner, toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag) - if raw == "" { - return map[string]any{} - } - parsed := parseMarkupInput(raw) - if parsed == nil { - return map[string]any{} - } - return parsed -} - -func parseFunctionCallTagStyle(text string) (ParsedToolCall, bool) { - m := functionCallPattern.FindStringSubmatch(text) - if len(m) < 2 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - input := map[string]any{} - for _, pm := range functionParamPattern.FindAllStringSubmatch(text, -1) { - if len(pm) < 3 { - continue - } - key := strings.TrimSpace(pm[1]) - val := extractRawTagValue(pm[2]) - if key != "" { - if parsed := parseStructuredToolCallInput(val); len(parsed) > 0 { - if isOnlyRawValue(parsed, val) { - input[key] = val - } else { - input[key] = parsed - } - } - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseAntmlFunctionCallStyles(text string) []ParsedToolCall { - matches := antmlFunctionCallPattern.FindAllStringSubmatch(text, -1) - if len(matches) == 0 { - return nil - } - out := make([]ParsedToolCall, 0, len(matches)) - for _, m := range matches { - if call, ok := parseSingleAntmlFunctionCallMatch(m); ok { - out = append(out, call) - } - } - if len(out) == 0 { - return nil - } - return out -} - -func parseSingleAntmlFunctionCallMatch(m []string) (ParsedToolCall, bool) { - if len(m) < 3 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - body := strings.TrimSpace(m[2]) - input := map[string]any{} - if strings.HasPrefix(body, "{") { - if err := json.Unmarshal([]byte(body), &input); err == nil { - return ParsedToolCall{Name: name, Input: input}, true - } - } - for _, am := range antmlArgumentPattern.FindAllStringSubmatch(body, -1) { - if len(am) < 3 { - continue - } - k := strings.TrimSpace(am[1]) - v := extractRawTagValue(am[2]) - if k != "" { - input[k] = v - } - } - if len(input) > 0 { - return ParsedToolCall{Name: name, Input: input}, true - } - if paramsRaw := findMarkupTagValue(body, toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); paramsRaw != "" { - if parsed := parseMarkupInput(paramsRaw); len(parsed) > 0 { - return ParsedToolCall{Name: name, Input: parsed}, true - } - } - if strings.Contains(body, "<") { - if parsed := parseStructuredToolCallInput(body); len(parsed) > 0 && !isOnlyRawValue(parsed, body) { - return ParsedToolCall{Name: name, Input: parsed}, true - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseInvokeFunctionCallStyle(text string) (ParsedToolCall, bool) { - m := invokeCallPattern.FindStringSubmatch(text) - if len(m) < 3 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - input := map[string]any{} - for _, pm := range invokeParamPattern.FindAllStringSubmatch(m[2], -1) { - if len(pm) < 3 { - continue - } - k := strings.TrimSpace(pm[1]) - v := extractRawTagValue(pm[2]) - if k != "" { - if parsed := parseStructuredToolCallInput(v); len(parsed) > 0 { - if isOnlyRawValue(parsed, v) { - input[k] = v - } else { - input[k] = parsed - } - } - } - } - if len(input) == 0 { - if argsRaw := findMarkupTagValue(m[2], toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); argsRaw != "" { - input = parseMarkupInput(argsRaw) - } else if kv := parseMarkupKVObject(m[2]); len(kv) > 0 { - input = kv - } else if parsed := parseStructuredToolCallInput(m[2]); len(parsed) > 0 && !isOnlyRawValue(parsed, strings.TrimSpace(html.UnescapeString(m[2]))) { - input = parsed - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseToolUseFunctionStyle(text string) (ParsedToolCall, bool) { - m := toolUseFunctionPattern.FindStringSubmatch(text) - if len(m) < 3 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - body := m[2] - input := map[string]any{} - for _, pm := range invokeParamPattern.FindAllStringSubmatch(body, -1) { - if len(pm) < 3 { - continue - } - k := strings.TrimSpace(pm[1]) - v := extractRawTagValue(pm[2]) - if k != "" { - if parsed := parseStructuredToolCallInput(v); len(parsed) > 0 { - if isOnlyRawValue(parsed, v) { - input[k] = v - } else { - input[k] = parsed - } - } - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseToolUseNameParametersStyle(text string) (ParsedToolCall, bool) { - m := toolUseNameParametersPattern.FindStringSubmatch(text) - if len(m) < 3 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - raw := strings.TrimSpace(m[2]) - input := map[string]any{} - if raw != "" { - if parsed := parseStructuredToolCallInput(raw); len(parsed) > 0 { - input = parsed - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseToolUseFunctionNameParametersStyle(text string) (ParsedToolCall, bool) { - m := toolUseFunctionNameParametersPattern.FindStringSubmatch(text) - if len(m) < 3 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - raw := strings.TrimSpace(m[2]) - input := map[string]any{} - if raw != "" { - if parsed := parseStructuredToolCallInput(raw); len(parsed) > 0 { - input = parsed - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseToolUseToolNameBodyStyle(text string) (ParsedToolCall, bool) { - m := toolUseToolNameBodyPattern.FindStringSubmatch(text) - if len(m) < 3 { - return ParsedToolCall{}, false - } - name := strings.TrimSpace(html.UnescapeString(m[1])) - if name == "" { - return ParsedToolCall{}, false - } - body := strings.TrimSpace(m[2]) - input := map[string]any{} - if body != "" { - if kv := parseXMLChildKV(body); len(kv) > 0 { - input = kv - } else if kv := parseMarkupKVObject(body); len(kv) > 0 { - input = kv - } else if parsed := parseStructuredToolCallInput(body); len(parsed) > 0 { - input = parsed - } - } - return ParsedToolCall{Name: name, Input: input}, true -} - -func parseXMLChildKV(body string) map[string]any { - trimmed := strings.TrimSpace(body) - if trimmed == "" { - return nil - } - parsed := parseStructuredToolCallInput(trimmed) - if len(parsed) == 0 { - return nil - } - return parsed + return ParsedToolCall{Name: name, Input: parseStructuredToolCallInput(m[2])}, true } func asString(v any) string { diff --git a/internal/toolcall/toolcalls_test.go b/internal/toolcall/toolcalls_test.go index ec1fa5b..25ee32c 100644 --- a/internal/toolcall/toolcalls_test.go +++ b/internal/toolcall/toolcalls_test.go @@ -16,8 +16,8 @@ func TestFormatOpenAIToolCalls(t *testing.T) { } } -func TestParseToolCallsSupportsClaudeXMLToolCall(t *testing.T) { - text := `Bashpwdshow cwd` +func TestParseToolCallsSupportsToolsWrapper(t *testing.T) { + text := `Bashpwdshow cwd` calls := ParseToolCalls(text, []string{"bash"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -30,10 +30,10 @@ func TestParseToolCallsSupportsClaudeXMLToolCall(t *testing.T) { } } -func TestParseToolCallsSupportsMultilineCDATAAndRepeatedXMLTags(t *testing.T) { - text := `write_filescript.shwrite_filescript.shfirstsecond` +]]>firstsecond` calls := ParseToolCalls(text, []string{"write_file"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -54,8 +54,8 @@ echo "hello" } } -func TestParseToolCallsSupportsCanonicalXMLParametersJSON(t *testing.T) { - text := `get_weather{"city":"beijing","unit":"c"}` +func TestParseToolCallsSupportsCanonicalParamsJSON(t *testing.T) { + text := `get_weather{"city":"beijing","unit":"c"}` calls := ParseToolCalls(text, []string{"get_weather"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -68,8 +68,8 @@ func TestParseToolCallsSupportsCanonicalXMLParametersJSON(t *testing.T) { } } -func TestParseToolCallsPreservesRawMalformedXMLParameters(t *testing.T) { - text := `execute_commandcd /root && git status` +func TestParseToolCallsPreservesRawMalformedParams(t *testing.T) { + text := `execute_commandcd /root && git status` calls := ParseToolCalls(text, []string{"execute_command"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -86,8 +86,8 @@ func TestParseToolCallsPreservesRawMalformedXMLParameters(t *testing.T) { } } -func TestParseToolCallsSupportsXMLParametersJSONWithAmpersandCommand(t *testing.T) { - text := `execute_command{"command":"sshpass -p 'xxx' ssh -o StrictHostKeyChecking=no -p 1111 root@111.111.111.111 'cd /root && git clone https://github.com/ericc-ch/copilot-api.git'","cwd":null,"timeout":null}` +func TestParseToolCallsSupportsParamsJSONWithAmpersandCommand(t *testing.T) { + text := `execute_command{"command":"sshpass -p 'xxx' ssh -o StrictHostKeyChecking=no -p 1111 root@111.111.111.111 'cd /root && git clone https://github.com/ericc-ch/copilot-api.git'","cwd":null,"timeout":null}` calls := ParseToolCalls(text, []string{"execute_command"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -101,8 +101,8 @@ func TestParseToolCallsSupportsXMLParametersJSONWithAmpersandCommand(t *testing. } } -func TestParseToolCallsDoesNotTreatParameterNameTagAsToolName(t *testing.T) { - text := `file.txtpwd` +func TestParseToolCallsDoesNotTreatParamsNameTagAsToolName(t *testing.T) { + text := `execute_commandfile.txtpwd` calls := ParseToolCalls(text, []string{"execute_command"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -110,13 +110,13 @@ func TestParseToolCallsDoesNotTreatParameterNameTagAsToolName(t *testing.T) { if calls[0].Name != "execute_command" { t.Fatalf("expected tool name execute_command, got %q", calls[0].Name) } - if calls[0].Input["name"] != "file.txt" { + if calls[0].Input["tool_name"] != "file.txt" { t.Fatalf("expected parameter name preserved, got %#v", calls[0].Input) } } -func TestParseToolCallsDetailedMarksXMLToolCallSyntax(t *testing.T) { - text := `Bashpwd` +func TestParseToolCallsDetailedMarksToolsSyntax(t *testing.T) { + text := `Bashpwd` res := ParseToolCallsDetailed(text, []string{"bash"}) if !res.SawToolCallSyntax { t.Fatalf("expected SawToolCallSyntax=true, got %#v", res) @@ -126,8 +126,8 @@ func TestParseToolCallsDetailedMarksXMLToolCallSyntax(t *testing.T) { } } -func TestParseToolCallsSupportsClaudeXMLJSONToolCall(t *testing.T) { - text := `{"tool":"Bash","params":{"command":"pwd","description":"show cwd"}}` +func TestParseToolCallsSupportsInlineJSONToolObject(t *testing.T) { + text := `{"name":"Bash","input":{"command":"pwd","description":"show cwd"}}` calls := ParseToolCalls(text, []string{"bash"}) if len(calls) != 1 { t.Fatalf("expected 1 call, got %#v", calls) @@ -140,170 +140,35 @@ func TestParseToolCallsSupportsClaudeXMLJSONToolCall(t *testing.T) { } } -func TestParseToolCallsSupportsFunctionCallTagStyle(t *testing.T) { - text := `Bashls -lalist` - calls := ParseToolCalls(text, []string{"bash"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "Bash" { - t.Fatalf("expected original tool name Bash, got %q", calls[0].Name) - } - if calls[0].Input["command"] != "ls -la" { - t.Fatalf("expected command argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsAntmlFunctionCallStyle(t *testing.T) { - text := `{"command":"pwd","description":"x"}` - calls := ParseToolCalls(text, []string{"bash"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "Bash" { - t.Fatalf("expected original tool name Bash, got %q", calls[0].Name) - } - if calls[0].Input["command"] != "pwd" { - t.Fatalf("expected command argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsAntmlArgumentStyle(t *testing.T) { - text := `pwdx` - calls := ParseToolCalls(text, []string{"bash"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "Bash" { - t.Fatalf("expected original tool name Bash, got %q", calls[0].Name) - } - if calls[0].Input["command"] != "pwd" { - t.Fatalf("expected command argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsInvokeFunctionCallStyle(t *testing.T) { - text := `pwdd` - calls := ParseToolCalls(text, []string{"bash"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "Bash" { - t.Fatalf("expected original tool name Bash, got %q", calls[0].Name) - } - if calls[0].Input["command"] != "pwd" { - t.Fatalf("expected command argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsToolUseFunctionParameterStyle(t *testing.T) { - text := `test` - calls := ParseToolCalls(text, []string{"search_web"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "search_web" { - t.Fatalf("expected canonical tool name search_web, got %q", calls[0].Name) - } - if calls[0].Input["query"] != "test" { - t.Fatalf("expected query argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsToolUseNameParametersStyle(t *testing.T) { - text := `write_file{"path":"/tmp/a.txt","content":"abc"}` - calls := ParseToolCalls(text, []string{"write_file"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "write_file" { - t.Fatalf("expected tool name write_file, got %q", calls[0].Name) - } - if calls[0].Input["path"] != "/tmp/a.txt" { - t.Fatalf("expected path argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsToolUseFunctionNameParametersStyle(t *testing.T) { - text := `write_file{"path":"/tmp/b.txt","content":"xyz"}` - calls := ParseToolCalls(text, []string{"write_file"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "write_file" { - t.Fatalf("expected tool name write_file, got %q", calls[0].Name) - } - if calls[0].Input["content"] != "xyz" { - t.Fatalf("expected content argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsToolUseToolNameBodyStyle(t *testing.T) { - text := `write_file/tmp/c.txthello` - calls := ParseToolCalls(text, []string{"write_file"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "write_file" { - t.Fatalf("expected tool name write_file, got %q", calls[0].Name) - } - if calls[0].Input["path"] != "/tmp/c.txt" { - t.Fatalf("expected path argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsNestedToolTagStyle(t *testing.T) { - text := `pwdshow cwd` - calls := ParseToolCalls(text, []string{"bash"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "Bash" { - t.Fatalf("expected original tool name Bash, got %q", calls[0].Name) - } - if calls[0].Input["command"] != "pwd" { - t.Fatalf("expected command argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsAntmlFunctionAttributeWithParametersTag(t *testing.T) { - text := `{"command":"pwd"}` - calls := ParseToolCalls(text, []string{"bash"}) - if len(calls) != 1 { - t.Fatalf("expected 1 call, got %#v", calls) - } - if calls[0].Name != "Bash" { - t.Fatalf("expected original tool name Bash, got %q", calls[0].Name) - } - if calls[0].Input["command"] != "pwd" { - t.Fatalf("expected command argument, got %#v", calls[0].Input) - } -} - -func TestParseToolCallsSupportsMultipleAntmlFunctionCalls(t *testing.T) { - text := `{"command":"pwd"}{"file_path":"README.md"}` - calls := ParseToolCalls(text, []string{"bash", "read"}) - if len(calls) != 2 { - t.Fatalf("expected 2 calls, got %#v", calls) - } - if calls[0].Name != "Bash" || calls[1].Name != "Read" { - t.Fatalf("expected original names [Bash Read], got %#v", calls) - } -} - func TestParseToolCallsDoesNotAcceptMismatchedMarkupTags(t *testing.T) { - text := `read_file{"path":"README.md"}` + text := `read_file{"path":"README.md"}` calls := ParseToolCalls(text, []string{"read_file"}) if len(calls) != 0 { t.Fatalf("expected mismatched tags to be rejected, got %#v", calls) } } -func TestParseToolCallsDoesNotTreatParametersFunctionNameAsToolName(t *testing.T) { - text := `data_onlyREADME.md` +func TestParseToolCallsDoesNotTreatNameInsideParamsAsToolName(t *testing.T) { + text := `data_onlyREADME.md` calls := ParseToolCalls(text, []string{"read_file"}) if len(calls) != 0 { - t.Fatalf("expected no tool call when function_name appears only under parameters, got %#v", calls) + t.Fatalf("expected no tool call when name appears only under params, got %#v", calls) + } +} + +func TestParseToolCallsRejectsLegacyToolCallsRoot(t *testing.T) { + text := `read_file{"path":"README.md"}` + calls := ParseToolCalls(text, []string{"read_file"}) + if len(calls) != 0 { + t.Fatalf("expected legacy tool_calls root to be rejected, got %#v", calls) + } +} + +func TestParseToolCallsRejectsLegacyParametersTag(t *testing.T) { + text := `read_file{"path":"README.md"}` + calls := ParseToolCalls(text, []string{"read_file"}) + if len(calls) != 0 { + t.Fatalf("expected legacy parameters tag to be rejected, got %#v", calls) } } @@ -445,7 +310,7 @@ func TestRepairLooseJSONWithNestedObjects(t *testing.T) { } func TestParseToolCallsUnescapesHTMLEntityArguments(t *testing.T) { - text := `Bash{"command":"echo a > out.txt"}` + text := `Bash{"command":"echo a > out.txt"}` calls := ParseToolCalls(text, []string{"bash"}) if len(calls) != 1 { t.Fatalf("expected one call, got %#v", calls) @@ -457,7 +322,7 @@ func TestParseToolCallsUnescapesHTMLEntityArguments(t *testing.T) { } func TestParseToolCallsIgnoresXMLInsideFencedCodeBlock(t *testing.T) { - text := "Here is an example:\n```xml\nread_file{\"path\":\"README.md\"}\n```\nDo not execute it." + text := "Here is an example:\n```xml\nread_file{\"path\":\"README.md\"}\n```\nDo not execute it." res := ParseToolCallsDetailed(text, []string{"read_file"}) if len(res.Calls) != 0 { t.Fatalf("expected no parsed calls for fenced example, got %#v", res.Calls) @@ -465,7 +330,7 @@ func TestParseToolCallsIgnoresXMLInsideFencedCodeBlock(t *testing.T) { } func TestParseToolCallsParsesOnlyNonFencedXMLToolCall(t *testing.T) { - text := "```xml\nread_file{\"path\":\"README.md\"}\n```\nsearch{\"q\":\"golang\"}" + text := "```xml\nread_file{\"path\":\"README.md\"}\n```\nsearch{\"q\":\"golang\"}" res := ParseToolCallsDetailed(text, []string{"read_file", "search"}) if len(res.Calls) != 1 { t.Fatalf("expected exactly one parsed call outside fence, got %#v", res.Calls) @@ -476,7 +341,7 @@ func TestParseToolCallsParsesOnlyNonFencedXMLToolCall(t *testing.T) { } func TestParseToolCallsParsesAfterFourBacktickFence(t *testing.T) { - text := "````markdown\n```xml\nread_file{\"path\":\"README.md\"}\n```\n````\nsearch{\"q\":\"outside\"}" + text := "````markdown\n```xml\nread_file{\"path\":\"README.md\"}\n```\n````\nsearch{\"q\":\"outside\"}" res := ParseToolCallsDetailed(text, []string{"read_file", "search"}) if len(res.Calls) != 1 { t.Fatalf("expected exactly one parsed call outside four-backtick fence, got %#v", res.Calls) diff --git a/tests/compat/expected/toolcalls_function_call_tag.json b/tests/compat/expected/toolcalls_function_call_tag.json index 5bcd9ce..4643a9b 100644 --- a/tests/compat/expected/toolcalls_function_call_tag.json +++ b/tests/compat/expected/toolcalls_function_call_tag.json @@ -1,13 +1,6 @@ { - "calls": [ - { - "name": "read_file", - "input": { - "path": "README.MD" - } - } - ], - "sawToolCallSyntax": true, + "calls": [], + "sawToolCallSyntax": false, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/compat/expected/toolcalls_invoke_attr.json b/tests/compat/expected/toolcalls_invoke_attr.json index 5bcd9ce..4643a9b 100644 --- a/tests/compat/expected/toolcalls_invoke_attr.json +++ b/tests/compat/expected/toolcalls_invoke_attr.json @@ -1,13 +1,6 @@ { - "calls": [ - { - "name": "read_file", - "input": { - "path": "README.MD" - } - } - ], - "sawToolCallSyntax": true, + "calls": [], + "sawToolCallSyntax": false, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/compat/expected/toolcalls_xml_tool_call.json b/tests/compat/expected/toolcalls_xml_tool_call.json index 5bcd9ce..4643a9b 100644 --- a/tests/compat/expected/toolcalls_xml_tool_call.json +++ b/tests/compat/expected/toolcalls_xml_tool_call.json @@ -1,13 +1,6 @@ { - "calls": [ - { - "name": "read_file", - "input": { - "path": "README.MD" - } - } - ], - "sawToolCallSyntax": true, + "calls": [], + "sawToolCallSyntax": false, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/compat/expected/toolcalls_xml_tool_name_parameters_json.json b/tests/compat/expected/toolcalls_xml_tool_name_parameters_json.json index 8eabce0..4643a9b 100644 --- a/tests/compat/expected/toolcalls_xml_tool_name_parameters_json.json +++ b/tests/compat/expected/toolcalls_xml_tool_name_parameters_json.json @@ -1,14 +1,6 @@ { - "calls": [ - { - "name": "get_weather", - "input": { - "city": "beijing", - "unit": "c" - } - } - ], - "sawToolCallSyntax": true, + "calls": [], + "sawToolCallSyntax": false, "rejectedByPolicy": false, "rejectedToolNames": [] } diff --git a/tests/compat/fixtures/toolcalls/function_call_tag.json b/tests/compat/fixtures/toolcalls/function_call_tag.json index 0f35956..a345ed2 100644 --- a/tests/compat/fixtures/toolcalls/function_call_tag.json +++ b/tests/compat/fixtures/toolcalls/function_call_tag.json @@ -1,5 +1,5 @@ { - "text": "read_file{\"path\":\"README.MD\"}", + "text": "read_file{\"path\":\"README.MD\"}", "tool_names": [ "read_file" ] diff --git a/tests/compat/fixtures/toolcalls/xml_tool_call.json b/tests/compat/fixtures/toolcalls/xml_tool_call.json index 279f1a2..b4ba281 100644 --- a/tests/compat/fixtures/toolcalls/xml_tool_call.json +++ b/tests/compat/fixtures/toolcalls/xml_tool_call.json @@ -1,5 +1,5 @@ { - "text": "read_file{\"path\":\"README.MD\"}", + "text": "read_file{\"path\":\"README.MD\"}", "tool_names": [ "read_file" ] diff --git a/tests/compat/fixtures/toolcalls/xml_tool_name_parameters_json.json b/tests/compat/fixtures/toolcalls/xml_tool_name_parameters_json.json index 6ccd51e..22843dc 100644 --- a/tests/compat/fixtures/toolcalls/xml_tool_name_parameters_json.json +++ b/tests/compat/fixtures/toolcalls/xml_tool_name_parameters_json.json @@ -1,5 +1,5 @@ { - "text": "get_weather{\"city\":\"beijing\",\"unit\":\"c\"}", + "text": "get_weather{\"city\":\"beijing\",\"unit\":\"c\"}", "tool_names": [ "get_weather" ] diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index a5f29ac..80b4bd9 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -42,7 +42,7 @@ test('extractToolNames keeps only declared tool names (Go parity)', () => { }); test('parseToolCalls parses XML markup tool call', () => { - const payload = 'read_file{"path":"README.MD"}'; + const payload = 'read_file{"path":"README.MD"}'; const calls = parseToolCalls(payload, ['read_file']); assert.equal(calls.length, 1); assert.equal(calls[0].name, 'read_file'); @@ -61,7 +61,7 @@ test('parseToolCalls ignores tool_call payloads that exist only inside fenced co const text = [ 'I will call a tool now.', '```xml', - 'read_file{"path":"README.md"}', + 'read_file{"path":"README.md"}', '```', ].join('\n'); const calls = parseToolCalls(text, ['read_file']); @@ -69,7 +69,7 @@ test('parseToolCalls ignores tool_call payloads that exist only inside fenced co }); test('parseToolCalls keeps unknown schema names when toolNames is provided', () => { - const payload = 'not_in_schema{"q":"go"}'; + const payload = 'not_in_schema{"q":"go"}'; const calls = parseToolCalls(payload, ['search']); assert.equal(calls.length, 1); assert.equal(calls[0].name, 'not_in_schema'); @@ -77,7 +77,7 @@ test('parseToolCalls keeps unknown schema names when toolNames is provided', () test('sieve emits tool_calls for XML tool call payload', () => { const events = runSieve( - ['read_file{"path":"README.MD"}'], + ['read_file{"path":"README.MD"}'], ['read_file'], ); const finalCalls = events.filter((evt) => evt.type === 'tool_calls').flatMap((evt) => evt.calls || []); @@ -88,8 +88,8 @@ test('sieve emits tool_calls for XML tool call payload', () => { test('sieve emits tool_calls when XML tag spans multiple chunks', () => { const events = runSieve( [ - 'read_file', - '{"path":"README.MD"}', + 'read_file', + '{"path":"README.MD"}', ], ['read_file'], ); @@ -103,10 +103,10 @@ test('sieve keeps long XML tool calls buffered until the closing tag arrives', ( const splitAt = longContent.length / 2; const events = runSieve( [ - '\n \n write_to_file\n \n \n \n write_to_file\n \n \n \n \n', + ']]>\n \n \n', ], ['write_to_file'], ); @@ -147,7 +147,7 @@ test('sieve keeps embedded invalid tool-like json as normal text to avoid stream }); test('sieve passes malformed executable-looking XML through as text', () => { - const chunk = '{"path":"README.MD"}'; + const chunk = '{"path":"README.MD"}'; const events = runSieve([chunk], ['read_file']); const leakedText = collectText(events); const hasToolCalls = events.some((evt) => evt.type === 'tool_calls' && evt.calls?.length > 0); @@ -159,14 +159,14 @@ test('sieve flushes incomplete captured XML tool blocks by falling back to raw t const events = runSieve( [ '前置正文G。', - '\n', + '\n', ' \n', ' read_file\n', ], ['read_file'], ); const leakedText = collectText(events); - const expected = ['前置正文G。', '\n', ' \n', ' read_file\n'].join(''); + const expected = ['前置正文G。', '\n', ' \n', ' read_file\n'].join(''); const hasToolCalls = events.some((evt) => evt.type === 'tool_calls' && evt.calls?.length > 0); assert.equal(hasToolCalls, false); assert.equal(leakedText, expected); @@ -176,7 +176,7 @@ test('sieve captures XML wrapper tags with attributes without leaking wrapper te const events = runSieve( [ '前置正文H。', - 'read_file{"path":"README.MD"}', + 'read_file{"path":"README.MD"}', '后置正文I。', ], ['read_file'], @@ -186,8 +186,8 @@ test('sieve captures XML wrapper tags with attributes without leaking wrapper te assert.equal(hasToolCall, true); assert.equal(leakedText.includes('前置正文H。'), true); assert.equal(leakedText.includes('后置正文I。'), true); - assert.equal(leakedText.includes(''), false); - assert.equal(leakedText.includes(''), false); + assert.equal(leakedText.includes(''), false); + assert.equal(leakedText.includes(''), false); }); test('sieve keeps plain text intact in tool mode when no tool call appears', () => { @@ -270,7 +270,7 @@ test('formatOpenAIStreamToolCalls reuses ids with the same idStore', () => { }); test('parseToolCalls rejects mismatched markup tags', () => { - const payload = 'read_file{"path":"README.md"}'; + const payload = 'read_file{"path":"README.md"}'; const calls = parseToolCalls(payload, ['read_file']); assert.equal(calls.length, 0); });