From 2bbf6031482a4e7737f1609880d1a7eb89e247d5 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Wed, 18 Mar 2026 00:52:24 +0800 Subject: [PATCH 01/12] fix: address PR #97 review findings --- internal/adapter/openai/handler_chat.go | 6 +- internal/admin/handler_accounts_testing.go | 14 ++++- .../admin/handler_accounts_testing_test.go | 57 +++++++++++++++++-- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/internal/adapter/openai/handler_chat.go b/internal/adapter/openai/handler_chat.go index c13e75c..c514e36 100644 --- a/internal/adapter/openai/handler_chat.go +++ b/internal/adapter/openai/handler_chat.go @@ -42,7 +42,9 @@ func (h *Handler) ChatCompletions(w http.ResponseWriter, r *http.Request) { // 2. 新请求可能获取到同一账号并开始使用 // 3. 异步删除仍在进行，会截断新请求正在使用的会话 if h.Store.AutoDeleteSessions() && a.DeepSeekToken != "" { - err := h.DS.DeleteAllSessionsForToken(context.Background(), a.DeepSeekToken) + deleteCtx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + err := h.DS.DeleteAllSessionsForToken(deleteCtx, a.DeepSeekToken) if err != nil { config.Logger.Warn("[auto_delete_sessions] failed", "account", a.AccountID, "error", err) } else { @@ -51,7 +53,7 @@ func (h *Handler) ChatCompletions(w http.ResponseWriter, r *http.Request) { } h.Auth.Release(a) }() - + r = r.WithContext(auth.WithAuth(r.Context(), a)) var req map[string]any diff --git a/internal/admin/handler_accounts_testing.go b/internal/admin/handler_accounts_testing.go index 0dc602d..93b0c8d 100644 --- a/internal/admin/handler_accounts_testing.go +++ b/internal/admin/handler_accounts_testing.go @@ -247,8 +247,18 @@ func (h *Handler) deleteAllSessions(w http.ResponseWriter, r *http.Request) { // 删除所有会话 err := h.DS.DeleteAllSessionsForToken(r.Context(), token) if err != nil { - writeJSON(w, http.StatusOK, map[string]any{"success": false, "message": "删除失败: " + err.Error()}) - return + // token 可能过期，尝试重新登录并重试一次 + newToken, loginErr := h.DS.Login(r.Context(), acc) + if loginErr != nil { + writeJSON(w, http.StatusOK, map[string]any{"success": false, "message": "删除失败: " + err.Error()}) + return + } + token = newToken + _ = h.Store.UpdateAccountToken(acc.Identifier(), token) + if retryErr := h.DS.DeleteAllSessionsForToken(r.Context(), token); retryErr != nil { + writeJSON(w, http.StatusOK, map[string]any{"success": false, "message": "删除失败: " + retryErr.Error()}) + return + } } writeJSON(w, http.StatusOK, map[string]any{"success": true, "message": "删除成功"}) diff --git a/internal/admin/handler_accounts_testing_test.go b/internal/admin/handler_accounts_testing_test.go index 1636669..e80eefe 100644 --- a/internal/admin/handler_accounts_testing_test.go +++ b/internal/admin/handler_accounts_testing_test.go @@ -1,9 +1,12 @@ package admin import ( + "bytes" "context" + "encoding/json" "errors" "net/http" + "net/http/httptest" "strings" "testing" @@ -13,10 +16,13 @@ import ( ) type testingDSMock struct { - loginCalls int - createSessionCalls int - getPowCalls int - callCompletionCalls int + loginCalls int + createSessionCalls int + getPowCalls int + callCompletionCalls int + deleteAllSessionsCalls int + deleteAllSessionsError error + deleteAllSessionsErrorOnce bool } func (m *testingDSMock) Login(_ context.Context, _ config.Account) (string, error) { @@ -40,6 +46,14 @@ func (m *testingDSMock) CallCompletion(_ context.Context, _ *auth.RequestAuth, _ } func (m *testingDSMock) DeleteAllSessionsForToken(_ context.Context, _ string) error { + m.deleteAllSessionsCalls++ + if m.deleteAllSessionsError != nil { + err := m.deleteAllSessionsError + if m.deleteAllSessionsErrorOnce { + m.deleteAllSessionsError = nil + } + return err + } return nil } @@ -83,3 +97,38 @@ func TestTestAccount_BatchModeOnlyCreatesSession(t *testing.T) { t.Fatalf("expected test status ok, got %q", updated.TestStatus) } } + +func TestDeleteAllSessions_RetryWithReloginOnDeleteFailure(t *testing.T) { + t.Setenv("DS2API_CONFIG_JSON", `{"accounts":[{"email":"batch@example.com","password":"pwd","token":"expired-token"}]}`) + store := config.LoadStore() + ds := &testingDSMock{deleteAllSessionsError: errors.New("token expired"), deleteAllSessionsErrorOnce: true} + h := &Handler{Store: store, DS: ds} + + req := httptest.NewRequest(http.MethodPost, "/delete-all", bytes.NewBufferString(`{"identifier":"batch@example.com"}`)) + rec := httptest.NewRecorder() + h.deleteAllSessions(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", rec.Code) + } + var resp map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if ok, _ := resp["success"].(bool); !ok { + t.Fatalf("expected success response, got %#v", resp) + } + if ds.loginCalls != 1 { + t.Fatalf("expected relogin once, got %d", ds.loginCalls) + } + if ds.deleteAllSessionsCalls != 2 { + t.Fatalf("expected delete called twice, got %d", ds.deleteAllSessionsCalls) + } + updated, ok := store.FindAccount("batch@example.com") + if !ok { + t.Fatal("expected account") + } + if updated.Token != "new-token" { + t.Fatalf("expected refreshed token persisted, got %q", updated.Token) + } +} From 52e7e7aae81b715628b6985affea7cd135c5453d Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 00:50:05 +0800 Subject: [PATCH 02/12] fix: unblock line gate and harden pow token recovery --- internal/deepseek/client_auth.go | 48 +++++++++---- internal/deepseek/client_session.go | 30 ++++---- internal/deepseek/client_session_delete.go | 9 ++- internal/util/toolcalls_json_repair.go | 79 ++++++++++++++++++++++ internal/util/toolcalls_parse.go | 74 -------------------- 5 files changed, 133 insertions(+), 107 deletions(-) create mode 100644 internal/util/toolcalls_json_repair.go diff --git a/internal/deepseek/client_auth.go b/internal/deepseek/client_auth.go index efa96ba..bedb75e 100644 --- a/internal/deepseek/client_auth.go +++ b/internal/deepseek/client_auth.go @@ -62,8 +62,8 @@ func (c *Client) CreateSession(ctx context.Context, a *auth.RequestAuth, maxAtte attempts++ continue } - code := intFrom(resp["code"]) - if status == http.StatusOK && code == 0 { + code, bizCode, msg, bizMsg := extractResponseStatus(resp) + if status == http.StatusOK && code == 0 && bizCode == 0 { data, _ := resp["data"].(map[string]any) bizData, _ := data["biz_data"].(map[string]any) sessionID, _ := bizData["id"].(string) @@ -71,10 +71,9 @@ func (c *Client) CreateSession(ctx context.Context, a *auth.RequestAuth, maxAtte return sessionID, nil } } - msg, _ := resp["msg"].(string) - config.Logger.Warn("[create_session] failed", "status", status, "code", code, "msg", msg, "use_config_token", a.UseConfigToken, "account", a.AccountID) + config.Logger.Warn("[create_session] failed", "status", status, "code", code, "biz_code", bizCode, "msg", msg, "biz_msg", bizMsg, "use_config_token", a.UseConfigToken, "account", a.AccountID) if a.UseConfigToken { - if isTokenInvalid(status, code, msg) && !refreshed { + if isTokenInvalid(status, code, bizCode, msg, bizMsg) && !refreshed { if c.Auth.RefreshToken(ctx, a) { refreshed = true continue @@ -96,6 +95,7 @@ func (c *Client) GetPow(ctx context.Context, a *auth.RequestAuth, maxAttempts in maxAttempts = c.maxRetries } attempts := 0 + refreshed := false for attempts < maxAttempts { headers := c.authHeaders(a.DeepSeekToken) resp, status, err := c.postJSONWithStatus(ctx, c.regular, DeepSeekCreatePowURL, headers, map[string]any{"target_path": "/api/v0/chat/completion"}) @@ -104,8 +104,8 @@ func (c *Client) GetPow(ctx context.Context, a *auth.RequestAuth, maxAttempts in attempts++ continue } - code := intFrom(resp["code"]) - if status == http.StatusOK && code == 0 { + code, bizCode, msg, bizMsg := extractResponseStatus(resp) + if status == http.StatusOK && code == 0 && bizCode == 0 { data, _ := resp["data"].(map[string]any) bizData, _ := data["biz_data"].(map[string]any) challenge, _ := bizData["challenge"].(map[string]any) @@ -116,15 +116,16 @@ func (c *Client) GetPow(ctx context.Context, a *auth.RequestAuth, maxAttempts in } return BuildPowHeader(challenge, answer) } - msg, _ := resp["msg"].(string) - config.Logger.Warn("[get_pow] failed", "status", status, "code", code, "msg", msg, "use_config_token", a.UseConfigToken, "account", a.AccountID) + config.Logger.Warn("[get_pow] failed", "status", status, "code", code, "biz_code", bizCode, "msg", msg, "biz_msg", bizMsg, "use_config_token", a.UseConfigToken, "account", a.AccountID) if a.UseConfigToken { - if isTokenInvalid(status, code, msg) { + if isTokenInvalid(status, code, bizCode, msg, bizMsg) && !refreshed { if c.Auth.RefreshToken(ctx, a) { + refreshed = true continue } } if c.Auth.SwitchAccount(ctx, a) { + refreshed = false attempts++ continue } @@ -143,15 +144,34 @@ func (c *Client) authHeaders(token string) map[string]string { return headers } -func isTokenInvalid(status int, code int, msg string) bool { - msg = strings.ToLower(msg) +func isTokenInvalid(status int, code int, bizCode int, msg string, bizMsg string) bool { + msg = strings.ToLower(strings.TrimSpace(msg) + " " + strings.TrimSpace(bizMsg)) if status == http.StatusUnauthorized || status == http.StatusForbidden { return true } - if code == 40001 || code == 40002 || code == 40003 { + if code == 40001 || code == 40002 || code == 40003 || bizCode == 40001 || bizCode == 40002 || bizCode == 40003 { return true } - return strings.Contains(msg, "token") || strings.Contains(msg, "unauthorized") + return strings.Contains(msg, "token") || + strings.Contains(msg, "unauthorized") || + strings.Contains(msg, "expired") || + strings.Contains(msg, "not login") || + strings.Contains(msg, "login required") || + strings.Contains(msg, "invalid jwt") +} + +func extractResponseStatus(resp map[string]any) (code int, bizCode int, msg string, bizMsg string) { + code = intFrom(resp["code"]) + msg, _ = resp["msg"].(string) + data, _ := resp["data"].(map[string]any) + bizCode = intFrom(data["biz_code"]) + bizMsg, _ = data["biz_msg"].(string) + if strings.TrimSpace(bizMsg) == "" { + if bizData, ok := data["biz_data"].(map[string]any); ok { + bizMsg, _ = bizData["msg"].(string) + } + } + return code, bizCode, msg, bizMsg } func normalizeMobileForLogin(raw string) (mobile string, areaCode any) { diff --git a/internal/deepseek/client_session.go b/internal/deepseek/client_session.go index fa422b7..f347ba6 100644 --- a/internal/deepseek/client_session.go +++ b/internal/deepseek/client_session.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "ds2api/internal/auth" "ds2api/internal/config" @@ -22,12 +23,12 @@ type SessionInfo struct { // SessionStats 会话统计结果 type SessionStats struct { - AccountID string // 账号标识 (email 或 mobile) - FirstPageCount int // 第一页会话数量（当 HasMore 为 true 时，真实总数可能更大） - PinnedCount int // 置顶会话数量 - HasMore bool // 是否还有更多页 - Success bool // 请求是否成功 - ErrorMessage string // 错误信息 + AccountID string // 账号标识 (email 或 mobile) + FirstPageCount int // 第一页会话数量（当 HasMore 为 true 时，真实总数可能更大） + PinnedCount int // 置顶会话数量 + HasMore bool // 是否还有更多页 + Success bool // 请求是否成功 + ErrorMessage string // 错误信息 } // GetSessionCount 获取单个账号的会话数量 @@ -56,8 +57,8 @@ func (c *Client) GetSessionCount(ctx context.Context, a *auth.RequestAuth, maxAt continue } - code := intFrom(resp["code"]) - if status == http.StatusOK && code == 0 { + code, bizCode, msg, bizMsg := extractResponseStatus(resp) + if status == http.StatusOK && code == 0 && bizCode == 0 { data, _ := resp["data"].(map[string]any) bizData, _ := data["biz_data"].(map[string]any) chatSessions, _ := bizData["chat_sessions"].([]any) @@ -79,12 +80,11 @@ func (c *Client) GetSessionCount(ctx context.Context, a *auth.RequestAuth, maxAt return stats, nil } - msg, _ := resp["msg"].(string) stats.ErrorMessage = fmt.Sprintf("status=%d, code=%d, msg=%s", status, code, msg) - config.Logger.Warn("[get_session_count] failed", "status", status, "code", code, "msg", msg, "account", a.AccountID) + config.Logger.Warn("[get_session_count] failed", "status", status, "code", code, "biz_code", bizCode, "msg", msg, "biz_msg", bizMsg, "account", a.AccountID) if a.UseConfigToken { - if isTokenInvalid(status, code, msg) && !refreshed { + if isTokenInvalid(status, code, bizCode, msg, bizMsg) && !refreshed { if c.Auth.RefreshToken(ctx, a) { refreshed = true continue @@ -114,9 +114,11 @@ func (c *Client) GetSessionCountForToken(ctx context.Context, token string) (*Se return nil, err } - code := intFrom(resp["code"]) - if status != http.StatusOK || code != 0 { - msg, _ := resp["msg"].(string) + code, bizCode, msg, bizMsg := extractResponseStatus(resp) + if status != http.StatusOK || code != 0 || bizCode != 0 { + if strings.TrimSpace(bizMsg) != "" { + msg = bizMsg + } return nil, fmt.Errorf("request failed: status=%d, code=%d, msg=%s", status, code, msg) } diff --git a/internal/deepseek/client_session_delete.go b/internal/deepseek/client_session_delete.go index e4814bf..06e32d1 100644 --- a/internal/deepseek/client_session_delete.go +++ b/internal/deepseek/client_session_delete.go @@ -49,18 +49,17 @@ func (c *Client) DeleteSession(ctx context.Context, a *auth.RequestAuth, session continue } - code := intFrom(resp["code"]) - if status == http.StatusOK && code == 0 { + code, bizCode, msg, bizMsg := extractResponseStatus(resp) + if status == http.StatusOK && code == 0 && bizCode == 0 { result.Success = true return result, nil } - msg, _ := resp["msg"].(string) result.ErrorMessage = fmt.Sprintf("status=%d, code=%d, msg=%s", status, code, msg) - config.Logger.Warn("[delete_session] failed", "status", status, "code", code, "msg", msg, "session_id", sessionID) + config.Logger.Warn("[delete_session] failed", "status", status, "code", code, "biz_code", bizCode, "msg", msg, "biz_msg", bizMsg, "session_id", sessionID) if a.UseConfigToken { - if isTokenInvalid(status, code, msg) && !refreshed { + if isTokenInvalid(status, code, bizCode, msg, bizMsg) && !refreshed { if c.Auth.RefreshToken(ctx, a) { refreshed = true continue diff --git a/internal/util/toolcalls_json_repair.go b/internal/util/toolcalls_json_repair.go new file mode 100644 index 0000000..6585d79 --- /dev/null +++ b/internal/util/toolcalls_json_repair.go @@ -0,0 +1,79 @@ +package util + +import ( + "regexp" + "strings" +) + +func repairInvalidJSONBackslashes(s string) string { + if !strings.Contains(s, "\\") { + return s + } + var out strings.Builder + out.Grow(len(s) + 10) + runes := []rune(s) + for i := 0; i < len(runes); i++ { + if runes[i] == '\\' { + if i+1 < len(runes) { + next := runes[i+1] + switch next { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + out.WriteRune('\\') + out.WriteRune(next) + i++ + continue + case 'u': + if i+5 < len(runes) { + isHex := true + for j := 1; j <= 4; j++ { + r := runes[i+1+j] + if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { + isHex = false + break + } + } + if isHex { + out.WriteRune('\\') + out.WriteRune('u') + for j := 1; j <= 4; j++ { + out.WriteRune(runes[i+1+j]) + } + i += 5 + continue + } + } + } + } + // Not a valid escape sequence, double it + out.WriteString("\\\\") + } else { + out.WriteRune(runes[i]) + } + } + return out.String() +} + +var unquotedKeyPattern = regexp.MustCompile(`([{,]\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*:`) + +// missingArrayBracketsPattern identifies a sequence of two or more JSON objects separated by commas +// that immediately follow a colon, which indicates a missing array bracket `[` `]`. +// E.g., "key": {"a": 1}, {"b": 2} -> "key": [{"a": 1}, {"b": 2}] +// NOTE: The pattern uses (?:[^{}]|\{[^{}]*\})* to support single-level nested {} objects, +// which handles cases like {"content": "x", "input": {"q": "y"}} +var missingArrayBracketsPattern = regexp.MustCompile(`(:\s*)(\{(?:[^{}]|\{[^{}]*\})*\}(?:\s*,\s*\{(?:[^{}]|\{[^{}]*\})*\})+)`) + +func RepairLooseJSON(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return s + } + // 1. Replace unquoted keys: {key: -> {"key": + s = unquotedKeyPattern.ReplaceAllString(s, `$1"$2":`) + + // 2. Heuristic: Fix missing array brackets for list of objects + // e.g., : {obj1}, {obj2} -> : [{obj1}, {obj2}] + // This specifically addresses DeepSeek's "list hallucination" + s = missingArrayBracketsPattern.ReplaceAllString(s, `$1[$2]`) + + return s +} diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 4830e97..726009c 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -2,7 +2,6 @@ package util import ( "encoding/json" - "regexp" "strings" ) @@ -298,76 +297,3 @@ func parseToolCallInput(v any) map[string]any { return map[string]any{} } } - -func repairInvalidJSONBackslashes(s string) string { - if !strings.Contains(s, "\\") { - return s - } - var out strings.Builder - out.Grow(len(s) + 10) - runes := []rune(s) - for i := 0; i < len(runes); i++ { - if runes[i] == '\\' { - if i+1 < len(runes) { - next := runes[i+1] - switch next { - case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': - out.WriteRune('\\') - out.WriteRune(next) - i++ - continue - case 'u': - if i+5 < len(runes) { - isHex := true - for j := 1; j <= 4; j++ { - r := runes[i+1+j] - if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { - isHex = false - break - } - } - if isHex { - out.WriteRune('\\') - out.WriteRune('u') - for j := 1; j <= 4; j++ { - out.WriteRune(runes[i+1+j]) - } - i += 5 - continue - } - } - } - } - // Not a valid escape sequence, double it - out.WriteString("\\\\") - } else { - out.WriteRune(runes[i]) - } - } - return out.String() -} - -var unquotedKeyPattern = regexp.MustCompile(`([{,]\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*:`) - -// missingArrayBracketsPattern identifies a sequence of two or more JSON objects separated by commas -// that immediately follow a colon, which indicates a missing array bracket `[` `]`. -// E.g., "key": {"a": 1}, {"b": 2} -> "key": [{"a": 1}, {"b": 2}] -// NOTE: The pattern uses (?:[^{}]|\{[^{}]*\})* to support single-level nested {} objects, -// which handles cases like {"content": "x", "input": {"q": "y"}} -var missingArrayBracketsPattern = regexp.MustCompile(`(:\s*)(\{(?:[^{}]|\{[^{}]*\})*\}(?:\s*,\s*\{(?:[^{}]|\{[^{}]*\})*\})+)`) - -func RepairLooseJSON(s string) string { - s = strings.TrimSpace(s) - if s == "" { - return s - } - // 1. Replace unquoted keys: {key: -> {"key": - s = unquotedKeyPattern.ReplaceAllString(s, `$1"$2":`) - - // 2. Heuristic: Fix missing array brackets for list of objects - // e.g., : {obj1}, {obj2} -> : [{obj1}, {obj2}] - // This specifically addresses DeepSeek's "list hallucination" - s = missingArrayBracketsPattern.ReplaceAllString(s, `$1[$2]`) - - return s -} From 2d5103997b849d31bcf0de11b42aa57d3e738e3d Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 01:15:15 +0800 Subject: [PATCH 03/12] fix(tool-sieve): keep mixed prose tool json in strict text mode --- .../js/helpers/stream-tool-sieve/sieve.js | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index a3b7fd8..ef12e8b 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -235,6 +235,17 @@ function consumeToolCapture(state, toolNames) { }; } + // Strict standalone mode: if the stream already produced meaningful prose, + // treat later tool-looking JSON as plain text instead of intercepting. + if ((state.recentTextTail || '').trim() !== '' && prefixPart.trim() === '') { + return { + ready: true, + prefix: captured, + calls: [], + suffix: '', + }; + } + const parsed = parseStandaloneToolCallsDetailed(captured.slice(actualStart, obj.end), toolNames); if (!Array.isArray(parsed.calls) || parsed.calls.length === 0) { if (parsed.sawToolCallSyntax && parsed.rejectedByPolicy) { @@ -253,6 +264,18 @@ function consumeToolCapture(state, toolNames) { }; } + // Strict standalone mode: only intercept when the tool payload stands alone + // (allowing only surrounding whitespace). If there is non-whitespace prose + // before/after the JSON object, keep everything as normal text. + if (prefixPart.trim() !== '' || suffixPart.trim() !== '') { + return { + ready: true, + prefix: captured, + calls: [], + suffix: '', + }; + } + return { ready: true, prefix: prefixPart, From 145501d4a5e5a973b21c29b9b5806e2c4f153f9c Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 01:15:32 +0800 Subject: [PATCH 04/12] fix(tool-sieve): allow mixed prose + tool json interception --- .../js/helpers/stream-tool-sieve/sieve.js | 26 +++---------------- tests/node/stream-tool-sieve.test.js | 12 ++++----- 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/internal/js/helpers/stream-tool-sieve/sieve.js b/internal/js/helpers/stream-tool-sieve/sieve.js index ef12e8b..ae95ffd 100644 --- a/internal/js/helpers/stream-tool-sieve/sieve.js +++ b/internal/js/helpers/stream-tool-sieve/sieve.js @@ -46,6 +46,9 @@ function processToolSieveChunk(state, chunk, toolNames) { if (Array.isArray(consumed.calls) && consumed.calls.length > 0) { state.pendingToolRaw = captured; state.pendingToolCalls = consumed.calls; + if (consumed.suffix) { + state.pending = consumed.suffix + state.pending; + } continue; } if (consumed.prefix) { @@ -235,17 +238,6 @@ function consumeToolCapture(state, toolNames) { }; } - // Strict standalone mode: if the stream already produced meaningful prose, - // treat later tool-looking JSON as plain text instead of intercepting. - if ((state.recentTextTail || '').trim() !== '' && prefixPart.trim() === '') { - return { - ready: true, - prefix: captured, - calls: [], - suffix: '', - }; - } - const parsed = parseStandaloneToolCallsDetailed(captured.slice(actualStart, obj.end), toolNames); if (!Array.isArray(parsed.calls) || parsed.calls.length === 0) { if (parsed.sawToolCallSyntax && parsed.rejectedByPolicy) { @@ -264,18 +256,6 @@ function consumeToolCapture(state, toolNames) { }; } - // Strict standalone mode: only intercept when the tool payload stands alone - // (allowing only surrounding whitespace). If there is non-whitespace prose - // before/after the JSON object, keep everything as normal text. - if (prefixPart.trim() !== '' || suffixPart.trim() !== '') { - return { - ready: true, - prefix: captured, - calls: [], - suffix: '', - }; - } - return { ready: true, prefix: prefixPart, diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index 61d72d6..7c703a1 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -259,28 +259,28 @@ test('sieve emits final tool_calls for split arguments payload without increment assert.deepEqual(finalCalls[0].input, { path: 'README.MD', mode: 'head' }); }); -test('sieve keeps tool json as text when leading prose exists (strict mode)', () => { +test('sieve still emits tool_calls when leading prose exists before tool json', () => { const events = runSieve( ['我将调用工具。', '{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}'], ['read_file'], ); const hasTool = events.some((evt) => (evt.type === 'tool_calls' && evt.calls?.length > 0) || (evt.type === 'tool_call_deltas' && evt.deltas?.length > 0)); const leakedText = collectText(events); - assert.equal(hasTool, false); + assert.equal(hasTool, true); assert.equal(leakedText.includes('我将调用工具。'), true); - assert.equal(leakedText.toLowerCase().includes('tool_calls'), true); + assert.equal(leakedText.toLowerCase().includes('tool_calls'), false); }); -test('sieve keeps same-chunk trailing prose payload as text in strict mode', () => { +test('sieve emits tool_calls and keeps trailing prose when payload and prose share a chunk', () => { const events = runSieve( ['{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}然后继续解释。'], ['read_file'], ); const hasTool = events.some((evt) => (evt.type === 'tool_calls' && evt.calls?.length > 0) || (evt.type === 'tool_call_deltas' && evt.deltas?.length > 0)); const leakedText = collectText(events); - assert.equal(hasTool, false); + assert.equal(hasTool, true); assert.equal(leakedText.includes('然后继续解释。'), true); - assert.equal(leakedText.toLowerCase().includes('tool_calls'), true); + assert.equal(leakedText.toLowerCase().includes('tool_calls'), false); }); test('formatOpenAIStreamToolCalls reuses ids with the same idStore', () => { From 4d549b710246b6f8d8d82e429fc08f2bece03b55 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 01:38:11 +0800 Subject: [PATCH 05/12] Revert "Merge branch 'dev' into codex/fix-issues-found-in-review" This reverts commit 33b0d1d144257fbf4bf341029d1f9d2d4a766f15, reversing changes made to efb484ba4fa725789d7d3634156de0843253617f. --- README.MD | 2 - TESTING.md | 7 - internal/util/toolcalls_parse.go | 69 ++++-- internal/util/toolcalls_parse_payload.go | 185 --------------- internal/util/toolcalls_repair.go | 276 ----------------------- internal/util/toolcalls_test.go | 31 +-- 6 files changed, 52 insertions(+), 518 deletions(-) delete mode 100644 internal/util/toolcalls_parse_payload.go delete mode 100644 internal/util/toolcalls_repair.go diff --git a/README.MD b/README.MD index 636e693..6b9383a 100644 --- a/README.MD +++ b/README.MD @@ -363,8 +363,6 @@ cp opencode.json.example opencode.json 3. 未在 `tools` 声明中的工具名会被严格拒绝，不会下发为有效 tool call 4. `responses` 支持并执行 `tool_choice`（`auto`/`none`/`required`/强制函数）；`required` 违规时非流式返回 `422`，流式返回 `response.failed` 5. 仅在通过策略校验后才会发出有效工具调用事件，避免错误工具名进入客户端执行链 -6. strict 模式下采用“可解析即拦截”：即使 tool JSON 前后混有 prose，只要结构可提取仍会拦截 tool_calls，剩余文本继续透传 -7. 当参数字符串无法可靠修复为对象时，会保留 `{"_raw":"..."}` 回退，避免 silent corruption ## 本地开发抓包工具 diff --git a/TESTING.md b/TESTING.md index bf821fe..8d1a309 100644 --- a/TESTING.md +++ b/TESTING.md @@ -200,13 +200,6 @@ go test -v -run 'TestParseToolCalls|TestRepair' ./internal/util/ # 2. 查看测试输出中的详细调试信息 go test -v -run TestParseToolCallsWithDeepSeekHallucination ./internal/util/ 2>&1 -# 2.1 strict 模式（Go/JS）语义对齐检查：混合 prose + tool JSON 仍可拦截 -node --test tests/node/stream-tool-sieve.test.js - -# 2.2 Windows 路径与文本换行语义回归 -go test -v -run TestParseToolCallsWithInvalidBackslashes ./internal/util/ -go test -v -run TestParseToolCallsWithPathEscapesAndTextNewlines ./internal/util/ - # 3. 检查具体测试用例的修复效果 # 测试用例位于 internal/util/toolcalls_test.go，包含： # - TestParseToolCallsWithDeepSeekHallucination: DeepSeek 典型幻觉输出 diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 82bb079..726009c 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -83,26 +83,31 @@ func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) return result } result.SawToolCallSyntax = looksLikeToolCallSyntax(trimmed) - - parsed := parseToolCallsPayload(trimmed) - if len(parsed) == 0 { - parsed = parseXMLToolCalls(trimmed) + candidates := []string{trimmed} + for _, candidate := range candidates { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + parsed := parseToolCallsPayload(candidate) + if len(parsed) == 0 { + parsed = parseXMLToolCalls(candidate) + } + if len(parsed) == 0 { + parsed = parseMarkupToolCalls(candidate) + } + if len(parsed) == 0 { + parsed = parseTextKVToolCalls(candidate) + } + if len(parsed) > 0 { + result.SawToolCallSyntax = true + calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) + result.Calls = calls + result.RejectedToolNames = rejectedNames + result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0 + return result + } } - if len(parsed) == 0 { - parsed = parseMarkupToolCalls(trimmed) - } - if len(parsed) == 0 { - parsed = parseTextKVToolCalls(trimmed) - } - if len(parsed) == 0 { - return result - } - - result.SawToolCallSyntax = true - calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) - result.Calls = calls - result.RejectedToolNames = rejectedNames - result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0 return result } @@ -135,7 +140,6 @@ func filterToolCallsDetailed(parsed []ParsedToolCall, availableToolNames []strin } return nil, rejected } - out := make([]ParsedToolCall, 0, len(parsed)) rejectedSet := map[string]struct{}{} rejected := make([]string, 0) @@ -164,6 +168,31 @@ func resolveAllowedToolName(name string, allowed map[string]struct{}, allowedCan return resolveAllowedToolNameWithLooseMatch(name, allowed, allowedCanonical) } +func parseToolCallsPayload(payload string) []ParsedToolCall { + var decoded any + if err := json.Unmarshal([]byte(payload), &decoded); err != nil { + // Try to repair backslashes first! Because LLMs often mix these two problems. + repaired := repairInvalidJSONBackslashes(payload) + // Try loose repair on top of that + repaired = RepairLooseJSON(repaired) + if err := json.Unmarshal([]byte(repaired), &decoded); err != nil { + return nil + } + } + switch v := decoded.(type) { + case map[string]any: + if tc, ok := v["tool_calls"]; ok { + return parseToolCallList(tc) + } + if parsed, ok := parseToolCallItem(v); ok { + return []ParsedToolCall{parsed} + } + case []any: + return parseToolCallList(v) + } + return nil +} + func looksLikeToolCallSyntax(text string) bool { lower := strings.ToLower(text) return strings.Contains(lower, "tool_calls") || diff --git a/internal/util/toolcalls_parse_payload.go b/internal/util/toolcalls_parse_payload.go deleted file mode 100644 index 5534e10..0000000 --- a/internal/util/toolcalls_parse_payload.go +++ /dev/null @@ -1,185 +0,0 @@ -package util - -import ( - "encoding/json" - "strings" -) - -func parseToolCallsPayload(payload string) []ParsedToolCall { - var decoded any - if err := json.Unmarshal([]byte(payload), &decoded); err != nil { - repaired := repairInvalidJSONBackslashesWithPathContext(payload) - repaired = RepairLooseJSON(repaired) - if err := json.Unmarshal([]byte(repaired), &decoded); err != nil { - return nil - } - } - - switch v := decoded.(type) { - case map[string]any: - if tc, ok := v["tool_calls"]; ok { - return parseToolCallList(tc) - } - if parsed, ok := parseToolCallItem(v); ok { - return []ParsedToolCall{parsed} - } - case []any: - return parseToolCallList(v) - } - return nil -} - -func parseToolCallList(v any) []ParsedToolCall { - items, ok := v.([]any) - if !ok { - return nil - } - out := make([]ParsedToolCall, 0, len(items)) - for _, item := range items { - m, ok := item.(map[string]any) - if !ok { - continue - } - if tc, ok := parseToolCallItem(m); ok { - out = append(out, tc) - } - } - if len(out) == 0 { - return nil - } - return out -} - -func parseToolCallItem(m map[string]any) (ParsedToolCall, bool) { - name, _ := m["name"].(string) - inputRaw, hasInput := m["input"] - - if fn, ok := m["function"].(map[string]any); ok { - if name == "" { - name, _ = fn["name"].(string) - } - if !hasInput { - if v, ok := fn["arguments"]; ok { - inputRaw = v - hasInput = true - } - } - } - if !hasInput { - for _, key := range []string{"arguments", "args", "parameters", "params"} { - if v, ok := m[key]; ok { - inputRaw = v - hasInput = true - break - } - } - } - if strings.TrimSpace(name) == "" { - return ParsedToolCall{}, false - } - return ParsedToolCall{ - Name: strings.TrimSpace(name), - Input: parseToolCallInput(inputRaw), - }, true -} - -func parseToolCallInput(v any) map[string]any { - switch x := v.(type) { - case nil: - return map[string]any{} - case map[string]any: - return x - case string: - raw := strings.TrimSpace(x) - if raw == "" { - return map[string]any{} - } - - if parsed := decodeJSONObject(raw); parsed != nil { - if hasSuspiciousPathControlChars(parsed) { - repaired := repairInvalidJSONBackslashesWithPathContext(raw) - if repaired != raw { - if reparsed := decodeJSONObject(repaired); reparsed != nil { - return reparsed - } - } - } - return parsed - } - - repaired := repairInvalidJSONBackslashesWithPathContext(raw) - if repaired != raw { - if reparsed := decodeJSONObject(repaired); reparsed != nil { - return reparsed - } - } - - repairedLoose := RepairLooseJSON(raw) - if repairedLoose != raw { - if reparsed := decodeJSONObject(repairedLoose); reparsed != nil { - return reparsed - } - } - return map[string]any{"_raw": raw} - default: - b, err := json.Marshal(x) - if err != nil { - return map[string]any{} - } - var parsed map[string]any - if err := json.Unmarshal(b, &parsed); err == nil && parsed != nil { - return parsed - } - return map[string]any{} - } -} - -func decodeJSONObject(raw string) map[string]any { - var parsed map[string]any - if err := json.Unmarshal([]byte(raw), &parsed); err == nil && parsed != nil { - return parsed - } - return nil -} - -func hasSuspiciousPathControlChars(v any) bool { - switch x := v.(type) { - case map[string]any: - for key, value := range x { - if isPathLikeKey(key) && hasControlCharsInString(value) { - return true - } - if hasSuspiciousPathControlChars(value) { - return true - } - } - case []any: - for _, item := range x { - if hasSuspiciousPathControlChars(item) { - return true - } - } - } - return false -} - -func isPathLikeKey(key string) bool { - lower := strings.ToLower(strings.TrimSpace(key)) - if lower == "" { - return false - } - for _, candidate := range []string{"path", "file", "filepath", "filename", "cwd", "dir", "directory"} { - if lower == candidate || strings.HasSuffix(lower, "_"+candidate) || strings.HasSuffix(lower, candidate+"_path") { - return true - } - } - return false -} - -func hasControlCharsInString(v any) bool { - s, ok := v.(string) - if !ok { - return false - } - return strings.ContainsAny(s, "\n\r\t") -} diff --git a/internal/util/toolcalls_repair.go b/internal/util/toolcalls_repair.go deleted file mode 100644 index 185cb45..0000000 --- a/internal/util/toolcalls_repair.go +++ /dev/null @@ -1,276 +0,0 @@ -package util - -import ( - "regexp" - "strings" -) - -var unquotedKeyPattern = regexp.MustCompile(`([{,]\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*:`) - -// fallback pattern for shallow objects; scanner-based repair runs first. -var missingArrayBracketsPattern = regexp.MustCompile(`(:\s*)(\{(?:[^{}]|\{[^{}]*\})*\}(?:\s*,\s*\{(?:[^{}]|\{[^{}]*\})*\})+)`) - -func repairInvalidJSONBackslashes(s string) string { - return repairInvalidJSONBackslashesWithPathContext(s) -} - -func repairInvalidJSONBackslashesWithPathContext(s string) string { - if !strings.Contains(s, "\\") { - return s - } - var out strings.Builder - out.Grow(len(s) + 10) - - runes := []rune(s) - pathKeyContext := buildPathKeyStringMask(runes) - inString := false - escaped := false - stringStart := -1 - - for i := 0; i < len(runes); i++ { - r := runes[i] - if r == '"' && !escaped { - inString = !inString - if inString { - stringStart = i - } else { - stringStart = -1 - } - out.WriteRune(r) - escaped = false - continue - } - if r == '\\' && inString { - if i+1 < len(runes) { - next := runes[i+1] - if next == 'u' { - if i+5 < len(runes) && isHex4(runes[i+2:i+6]) { - out.WriteRune('\\') - out.WriteRune('u') - for _, hx := range runes[i+2 : i+6] { - out.WriteRune(hx) - } - i += 5 - escaped = false - continue - } - } else if shouldKeepEscape(next, pathKeyContext[stringStart]) { - out.WriteRune('\\') - out.WriteRune(next) - i++ - escaped = false - continue - } - } - out.WriteString("\\\\") - escaped = false - continue - } - out.WriteRune(r) - escaped = r == '\\' && !escaped - if r != '\\' { - escaped = false - } - } - return out.String() -} - -func shouldKeepEscape(next rune, inPathContext bool) bool { - switch next { - case '"', '\\', '/', 'b', 'f': - return true - case 'n', 'r', 't': - return !inPathContext - case 'u': - return true - default: - return false - } -} - -func buildPathKeyStringMask(runes []rune) map[int]bool { - mask := map[int]bool{} - inString := false - escaped := false - stringStart := -1 - var lastKey string - - for i := 0; i < len(runes); i++ { - r := runes[i] - if !inString { - if r == '"' { - inString = true - stringStart = i - } - continue - } - if escaped { - escaped = false - continue - } - if r == '\\' { - escaped = true - continue - } - if r != '"' { - continue - } - - value := string(runes[stringStart+1 : i]) - j := i + 1 - for j < len(runes) && (runes[j] == ' ' || runes[j] == '\n' || runes[j] == '\r' || runes[j] == '\t') { - j++ - } - if j < len(runes) && runes[j] == ':' { - lastKey = strings.ToLower(strings.TrimSpace(value)) - } else if isPathLikeKey(lastKey) { - mask[stringStart] = true - } - - inString = false - stringStart = -1 - } - return mask -} - -func RepairLooseJSON(s string) string { - s = strings.TrimSpace(s) - if s == "" { - return s - } - s = unquotedKeyPattern.ReplaceAllString(s, `$1"$2":`) - s = repairMissingArrayBracketsByScanner(s) - return missingArrayBracketsPattern.ReplaceAllString(s, `$1[$2]`) -} - -func repairMissingArrayBracketsByScanner(s string) string { - const maxScanLen = 200_000 - if len(s) == 0 || len(s) > maxScanLen { - return s - } - - var out strings.Builder - out.Grow(len(s) + 8) - i := 0 - for i < len(s) { - if s[i] != ':' { - out.WriteByte(s[i]) - i++ - continue - } - out.WriteByte(':') - i++ - for i < len(s) && isJSONWhitespace(s[i]) { - out.WriteByte(s[i]) - i++ - } - if i >= len(s) || s[i] != '{' { - continue - } - - start := i - end := scanJSONObjectEnd(s, start) - if end < 0 { - out.WriteString(s[start:]) - break - } - cursor := end - next := skipJSONWhitespace(s, cursor) - if next >= len(s) || s[next] != ',' { - out.WriteString(s[start:end]) - i = end - continue - } - - seqEnd := end - hasMultiple := false - for { - comma := skipJSONWhitespace(s, seqEnd) - if comma >= len(s) || s[comma] != ',' { - break - } - objStart := skipJSONWhitespace(s, comma+1) - if objStart >= len(s) || s[objStart] != '{' { - break - } - objEnd := scanJSONObjectEnd(s, objStart) - if objEnd < 0 { - break - } - hasMultiple = true - seqEnd = objEnd - } - if !hasMultiple { - out.WriteString(s[start:end]) - i = end - continue - } - - out.WriteByte('[') - out.WriteString(s[start:seqEnd]) - out.WriteByte(']') - i = seqEnd - } - return out.String() -} - -func scanJSONObjectEnd(s string, start int) int { - depth := 0 - inString := false - escaped := false - for i := start; i < len(s); i++ { - c := s[i] - if inString { - if escaped { - escaped = false - continue - } - if c == '\\' { - escaped = true - continue - } - if c == '"' { - inString = false - } - continue - } - if c == '"' { - inString = true - continue - } - if c == '{' { - depth++ - continue - } - if c == '}' { - depth-- - if depth == 0 { - return i + 1 - } - } - } - return -1 -} - -func skipJSONWhitespace(s string, i int) int { - for i < len(s) && isJSONWhitespace(s[i]) { - i++ - } - return i -} - -func isJSONWhitespace(b byte) bool { - return b == ' ' || b == '\n' || b == '\r' || b == '\t' -} - -func isHex4(seq []rune) bool { - if len(seq) != 4 { - return false - } - for _, r := range seq { - if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { - return false - } - } - return true -} diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index 10458df..e3fae5d 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -288,7 +288,7 @@ func TestRepairInvalidJSONBackslashes(t *testing.T) { input string expected string }{ - {`{"path": "C:\Users\name"}`, `{"path": "C:\\Users\\name"}`}, + {`{"path": "C:\Users\name"}`, `{"path": "C:\\Users\name"}`}, {`{"cmd": "cd D:\git_codes"}`, `{"cmd": "cd D:\\git_codes"}`}, {`{"text": "line1\nline2"}`, `{"text": "line1\nline2"}`}, {`{"path": "D:\\back\\slash"}`, `{"path": "D:\\back\\slash"}`}, @@ -419,29 +419,9 @@ func TestParseToolCallsWithMixedWindowsPaths(t *testing.T) { } } -func TestParseToolCallsWithPathEscapesAndTextNewlines(t *testing.T) { - text := `{"name":"write_file","input":"{\"content\":\"line1\\nline2\",\"path\":\"D:\\tmp\\a.txt\"}"}` - availableTools := []string{"write_file"} - parsed := ParseToolCalls(text, availableTools) - if len(parsed) != 1 { - t.Fatalf("expected 1 parsed tool call, got %d", len(parsed)) - } - - content, _ := parsed[0].Input["content"].(string) - path, _ := parsed[0].Input["path"].(string) - if !strings.Contains(content, "line1\nline2") { - t.Fatalf("expected content to preserve newline semantics, got %q", content) - } - if strings.ContainsAny(path, "\n\r\t") { - t.Fatalf("expected path to avoid control chars, got %q", path) - } - if !strings.Contains(path, `D:\tmp\a.txt`) { - t.Fatalf("expected path with literal backslashes, got %q", path) - } -} - func TestRepairLooseJSONWithNestedObjects(t *testing.T) { - // 覆盖深层嵌套对象的方括号修复，避免 regex 单层能力带来的漂移。 + // 测试嵌套对象的修复：DeepSeek 幻觉输出，每个元素内部包含嵌套 {} + // 注意：正则只支持单层嵌套，不支持更深层次的嵌套 tests := []struct { name string input string @@ -507,11 +487,6 @@ func TestRepairLooseJSONWithNestedObjects(t *testing.T) { input: `"tasks": {"id":1}, {"id":2}, {"id":3}, {"id":4}, {"id":5}`, expected: `"tasks": [{"id":1}, {"id":2}, {"id":3}, {"id":4}, {"id":5}]`, }, - { - name: "深层嵌套对象", - input: `"todos": {"meta":{"a":{"b":1}},"content":"x"}, {"meta":{"a":{"b":2}},"content":"y"}`, - expected: `"todos": [{"meta":{"a":{"b":1}},"content":"x"}, {"meta":{"a":{"b":2}},"content":"y"}]`, - }, } for _, tt := range tests { From 20b603666dcfd47191e56968c070545938a2b650 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:03:32 +0800 Subject: [PATCH 06/12] Allow standalone parser to detect mixed prose tool JSON --- .../adapter/openai/handler_toolcall_test.go | 15 ++++----- .../adapter/openai/responses_stream_test.go | 8 +++-- internal/adapter/openai/stream_status_test.go | 8 ++--- internal/format/openai/render_test.go | 12 +++---- internal/util/toolcalls_parse.go | 31 ++++++++++++------- internal/util/toolcalls_test.go | 6 ++-- .../toolcalls_standalone_mixed_prose.json | 11 +++++-- 7 files changed, 55 insertions(+), 36 deletions(-) diff --git a/internal/adapter/openai/handler_toolcall_test.go b/internal/adapter/openai/handler_toolcall_test.go index 0d85afd..00a0e8d 100644 --- a/internal/adapter/openai/handler_toolcall_test.go +++ b/internal/adapter/openai/handler_toolcall_test.go @@ -211,7 +211,7 @@ func TestHandleNonStreamUnknownToolNotIntercepted(t *testing.T) { } } -func TestHandleNonStreamEmbeddedToolCallExampleRemainsText(t *testing.T) { +func TestHandleNonStreamEmbeddedToolCallExamplePromotesToolCall(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( `data: {"p":"response/content","v":"下面是示例："}`, @@ -229,16 +229,17 @@ func TestHandleNonStreamEmbeddedToolCallExampleRemainsText(t *testing.T) { out := decodeJSONBody(t, rec.Body.String()) choices, _ := out["choices"].([]any) choice, _ := choices[0].(map[string]any) - if choice["finish_reason"] != "stop" { - t.Fatalf("expected finish_reason=stop, got %#v", choice["finish_reason"]) + if choice["finish_reason"] != "tool_calls" { + t.Fatalf("expected finish_reason=tool_calls, got %#v", choice["finish_reason"]) } msg, _ := choice["message"].(map[string]any) - if _, ok := msg["tool_calls"]; ok { - t.Fatalf("did not expect tool_calls field for embedded example: %#v", msg["tool_calls"]) + toolCalls, _ := msg["tool_calls"].([]any) + if len(toolCalls) != 1 { + t.Fatalf("expected one tool_call field for embedded example: %#v", msg["tool_calls"]) } content, _ := msg["content"].(string) - if !strings.Contains(content, "下面是示例：") || !strings.Contains(content, "请勿执行。") || !strings.Contains(content, `"tool_calls"`) { - t.Fatalf("expected embedded example to remain plain text, got %#v", content) + if strings.Contains(content, `"tool_calls"`) { + t.Fatalf("expected raw tool_calls json stripped from content, got %#v", content) } } diff --git a/internal/adapter/openai/responses_stream_test.go b/internal/adapter/openai/responses_stream_test.go index f62ff13..7d15ede 100644 --- a/internal/adapter/openai/responses_stream_test.go +++ b/internal/adapter/openai/responses_stream_test.go @@ -297,7 +297,7 @@ func TestHandleResponsesStreamOutputTextDeltaCarriesItemIndexes(t *testing.T) { } } -func TestHandleResponsesStreamThinkingAndMixedToolExampleRemainMessageOnly(t *testing.T) { +func TestHandleResponsesStreamThinkingAndMixedToolExampleEmitsFunctionCall(t *testing.T) { h := &Handler{} req := httptest.NewRequest(http.MethodPost, "/v1/responses", nil) rec := httptest.NewRecorder() @@ -333,6 +333,7 @@ func TestHandleResponsesStreamThinkingAndMixedToolExampleRemainMessageOnly(t *te responseObj, _ := completedPayload["response"].(map[string]any) output, _ := responseObj["output"].([]any) hasMessage := false + hasFunctionCall := false for _, item := range output { m, _ := item.(map[string]any) if m == nil { @@ -342,12 +343,15 @@ func TestHandleResponsesStreamThinkingAndMixedToolExampleRemainMessageOnly(t *te hasMessage = true } if asString(m["type"]) == "function_call" { - t.Fatalf("did not expect function_call output for mixed prose tool example, output=%#v", output) + hasFunctionCall = true } } if !hasMessage { t.Fatalf("expected message output for mixed prose tool example, output=%#v", output) } + if !hasFunctionCall { + t.Fatalf("expected function_call output for mixed prose tool example, output=%#v", output) + } } func TestHandleResponsesStreamToolChoiceNoneRejectsFunctionCall(t *testing.T) { diff --git a/internal/adapter/openai/stream_status_test.go b/internal/adapter/openai/stream_status_test.go index 033dc37..c76d881 100644 --- a/internal/adapter/openai/stream_status_test.go +++ b/internal/adapter/openai/stream_status_test.go @@ -171,15 +171,15 @@ func TestResponsesNonStreamMixedProseToolPayloadHandlerPath(t *testing.T) { t.Fatalf("decode response failed: %v body=%s", err, rec.Body.String()) } outputText, _ := out["output_text"].(string) - if outputText == "" { - t.Fatalf("expected output_text preserved for mixed prose payload") + if outputText != "" { + t.Fatalf("expected output_text hidden for mixed prose tool payload, got %q", outputText) } output, _ := out["output"].([]any) if len(output) != 1 { t.Fatalf("expected one output item, got %#v", output) } first, _ := output[0].(map[string]any) - if first["type"] != "message" { - t.Fatalf("expected message output item, got %#v", output) + if first["type"] != "function_call" { + t.Fatalf("expected function_call output item, got %#v", output) } } diff --git a/internal/format/openai/render_test.go b/internal/format/openai/render_test.go index 7a9d897..2ec05c6 100644 --- a/internal/format/openai/render_test.go +++ b/internal/format/openai/render_test.go @@ -45,7 +45,7 @@ func TestBuildResponseObjectToolCallsFollowChatShape(t *testing.T) { } } -func TestBuildResponseObjectTreatsMixedProseToolPayloadAsText(t *testing.T) { +func TestBuildResponseObjectPromotesMixedProseToolPayloadToFunctionCall(t *testing.T) { obj := BuildResponseObject( "resp_test", "gpt-4o", @@ -56,16 +56,16 @@ func TestBuildResponseObjectTreatsMixedProseToolPayloadAsText(t *testing.T) { ) outputText, _ := obj["output_text"].(string) - if outputText == "" { - t.Fatalf("expected output_text preserved for mixed prose payload") + if outputText != "" { + t.Fatalf("expected output_text hidden for mixed prose tool payload, got %q", outputText) } output, _ := obj["output"].([]any) if len(output) != 1 { - t.Fatalf("expected one message output item, got %#v", obj["output"]) + t.Fatalf("expected one function_call output item, got %#v", obj["output"]) } first, _ := output[0].(map[string]any) - if first["type"] != "message" { - t.Fatalf("expected message output type, got %#v", first["type"]) + if first["type"] != "function_call" { + t.Fatalf("expected function_call output type, got %#v", first["type"]) } } diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 726009c..e55861b 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -75,21 +75,19 @@ func ParseStandaloneToolCalls(text string, availableToolNames []string) []Parsed func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) ToolCallParseResult { result := ToolCallParseResult{} - trimmed := strings.TrimSpace(text) + trimmed := strings.TrimSpace(stripFencedCodeBlocks(text)) if trimmed == "" { return result } - if looksLikeToolExampleContext(trimmed) { - return result - } result.SawToolCallSyntax = looksLikeToolCallSyntax(trimmed) - candidates := []string{trimmed} + candidates := buildToolCallCandidates(trimmed) + var parsed []ParsedToolCall for _, candidate := range candidates { candidate = strings.TrimSpace(candidate) if candidate == "" { continue } - parsed := parseToolCallsPayload(candidate) + parsed = parseToolCallsPayload(candidate) if len(parsed) == 0 { parsed = parseXMLToolCalls(candidate) } @@ -100,14 +98,23 @@ func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) parsed = parseTextKVToolCalls(candidate) } if len(parsed) > 0 { - result.SawToolCallSyntax = true - calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) - result.Calls = calls - result.RejectedToolNames = rejectedNames - result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0 - return result + break } } + if len(parsed) == 0 { + parsed = parseXMLToolCalls(trimmed) + if len(parsed) == 0 { + parsed = parseTextKVToolCalls(trimmed) + if len(parsed) == 0 { + return result + } + } + } + result.SawToolCallSyntax = true + calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames) + result.Calls = calls + result.RejectedToolNames = rejectedNames + result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0 return result } diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index e3fae5d..da6e59a 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -99,10 +99,10 @@ func TestFormatOpenAIToolCalls(t *testing.T) { } } -func TestParseStandaloneToolCallsOnlyMatchesStandalonePayload(t *testing.T) { +func TestParseStandaloneToolCallsSupportsMixedProsePayload(t *testing.T) { mixed := `这里是示例：{"tool_calls":[{"name":"search","input":{"q":"go"}}]}` - if calls := ParseStandaloneToolCalls(mixed, []string{"search"}); len(calls) != 0 { - t.Fatalf("expected standalone parser to ignore mixed prose, got %#v", calls) + if calls := ParseStandaloneToolCalls(mixed, []string{"search"}); len(calls) != 1 { + t.Fatalf("expected standalone parser to parse mixed prose payload, got %#v", calls) } standalone := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}` diff --git a/tests/compat/expected/toolcalls_standalone_mixed_prose.json b/tests/compat/expected/toolcalls_standalone_mixed_prose.json index 0fcce27..124de59 100644 --- a/tests/compat/expected/toolcalls_standalone_mixed_prose.json +++ b/tests/compat/expected/toolcalls_standalone_mixed_prose.json @@ -1,6 +1,13 @@ { - "calls": [], + "calls": [ + { + "name": "read_file", + "input": { + "path": "README.MD" + } + } + ], "sawToolCallSyntax": true, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} From fb5fc0e8855cc91ba212186e48ac8537b38a97bc Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:03:46 +0800 Subject: [PATCH 07/12] Default to aggressive tool-call interception in mixed/fenced text --- .../adapter/claude/handler_stream_test.go | 29 +++++++----- .../adapter/openai/handler_toolcall_test.go | 29 ++++++------ internal/adapter/openai/tool_sieve_core.go | 44 ++++++------------- .../adapter/openai/tool_sieve_incremental.go | 3 -- internal/format/openai/render_test.go | 12 ++--- internal/util/toolcalls_parse.go | 6 +-- internal/util/toolcalls_test.go | 10 ++--- internal/util/util_edge_test.go | 4 +- .../expected/toolcalls_fenced_json.json | 13 ++++-- .../toolcalls_standalone_fenced_example.json | 13 ++++-- 10 files changed, 79 insertions(+), 84 deletions(-) diff --git a/internal/adapter/claude/handler_stream_test.go b/internal/adapter/claude/handler_stream_test.go index dda425a..77e62c8 100644 --- a/internal/adapter/claude/handler_stream_test.go +++ b/internal/adapter/claude/handler_stream_test.go @@ -358,7 +358,7 @@ func TestHandleClaudeStreamRealtimeToolSafetyAcrossStructuredFormats(t *testing. } } -func TestHandleClaudeStreamRealtimeDoesNotStopOnUnclosedFencedToolExample(t *testing.T) { +func TestHandleClaudeStreamRealtimePromotesUnclosedFencedToolExample(t *testing.T) { h := &Handler{} resp := makeClaudeSSEHTTPResponse( "data: {\"p\":\"response/content\",\"v\":\"Here is an example:\\n```json\\n{\\\"tool_calls\\\":[{\\\"name\\\":\\\"Bash\\\",\\\"input\\\":{\\\"command\\\":\\\"pwd\\\"}}]}\"}", @@ -371,22 +371,27 @@ func TestHandleClaudeStreamRealtimeDoesNotStopOnUnclosedFencedToolExample(t *tes h.handleClaudeStreamRealtime(rec, req, resp, "claude-sonnet-4-5", []any{map[string]any{"role": "user", "content": "show example only"}}, false, false, []string{"Bash"}) frames := parseClaudeFrames(t, rec.Body.String()) + foundToolUse := false for _, f := range findClaudeFrames(frames, "content_block_start") { contentBlock, _ := f.Payload["content_block"].(map[string]any) if contentBlock["type"] == "tool_use" { - t.Fatalf("unexpected tool_use for fenced example, body=%s", rec.Body.String()) - } - } - - foundEndTurn := false - for _, f := range findClaudeFrames(frames, "message_delta") { - delta, _ := f.Payload["delta"].(map[string]any) - if delta["stop_reason"] == "end_turn" { - foundEndTurn = true + foundToolUse = true break } } - if !foundEndTurn { - t.Fatalf("expected stop_reason=end_turn, body=%s", rec.Body.String()) + if !foundToolUse { + t.Fatalf("expected tool_use for fenced example, body=%s", rec.Body.String()) + } + + foundToolStop := false + for _, f := range findClaudeFrames(frames, "message_delta") { + delta, _ := f.Payload["delta"].(map[string]any) + if delta["stop_reason"] == "tool_use" { + foundToolStop = true + break + } + } + if !foundToolStop { + t.Fatalf("expected stop_reason=tool_use, body=%s", rec.Body.String()) } } diff --git a/internal/adapter/openai/handler_toolcall_test.go b/internal/adapter/openai/handler_toolcall_test.go index 00a0e8d..ef22803 100644 --- a/internal/adapter/openai/handler_toolcall_test.go +++ b/internal/adapter/openai/handler_toolcall_test.go @@ -243,7 +243,7 @@ func TestHandleNonStreamEmbeddedToolCallExamplePromotesToolCall(t *testing.T) { } } -func TestHandleNonStreamFencedToolCallExampleNotIntercepted(t *testing.T) { +func TestHandleNonStreamFencedToolCallExamplePromotesToolCall(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( "data: {\"p\":\"response/content\",\"v\":\"```json\\n{\\\"tool_calls\\\":[{\\\"name\\\":\\\"search\\\",\\\"input\\\":{\\\"q\\\":\\\"go\\\"}}]}\\n```\"}", @@ -259,16 +259,17 @@ func TestHandleNonStreamFencedToolCallExampleNotIntercepted(t *testing.T) { out := decodeJSONBody(t, rec.Body.String()) choices, _ := out["choices"].([]any) choice, _ := choices[0].(map[string]any) - if choice["finish_reason"] != "stop" { - t.Fatalf("expected finish_reason=stop, got %#v", choice["finish_reason"]) + if choice["finish_reason"] != "tool_calls" { + t.Fatalf("expected finish_reason=tool_calls, got %#v", choice["finish_reason"]) } msg, _ := choice["message"].(map[string]any) - if _, ok := msg["tool_calls"]; ok { - t.Fatalf("did not expect tool_calls field for fenced example: %#v", msg["tool_calls"]) + toolCalls, _ := msg["tool_calls"].([]any) + if len(toolCalls) != 1 { + t.Fatalf("expected one tool_call field for fenced example: %#v", msg["tool_calls"]) } content, _ := msg["content"].(string) - if !strings.Contains(content, "```json") || !strings.Contains(content, `"tool_calls"`) { - t.Fatalf("expected fenced tool example to pass through as text, got %q", content) + if strings.Contains(content, `"tool_calls"`) { + t.Fatalf("expected raw tool_calls json stripped from content, got %q", content) } } @@ -616,7 +617,7 @@ func TestHandleStreamToolCallWithSameChunkTrailingTextRemainsText(t *testing.T) } } -func TestHandleStreamFencedToolCallSnippetRemainsText(t *testing.T) { +func TestHandleStreamFencedToolCallSnippetPromotesToolCall(t *testing.T) { h := &Handler{} resp := makeSSEHTTPResponse( fmt.Sprintf(`data: {"p":"response/content","v":%q}`, "下面是调用示例：\n```json\n"), @@ -632,8 +633,8 @@ func TestHandleStreamFencedToolCallSnippetRemainsText(t *testing.T) { if !done { t.Fatalf("expected [DONE], body=%s", rec.Body.String()) } - if streamHasToolCallsDelta(frames) { - t.Fatalf("did not expect tool_calls delta for fenced snippet, body=%s", rec.Body.String()) + if !streamHasToolCallsDelta(frames) { + t.Fatalf("expected tool_calls delta for fenced snippet, body=%s", rec.Body.String()) } content := strings.Builder{} for _, frame := range frames { @@ -647,11 +648,11 @@ func TestHandleStreamFencedToolCallSnippetRemainsText(t *testing.T) { } } got := content.String() - if !strings.Contains(got, "```json") || !strings.Contains(strings.ToLower(got), "tool_calls") { - t.Fatalf("expected fenced tool snippet in content, got=%q", got) + if strings.Contains(strings.ToLower(got), "tool_calls") { + t.Fatalf("expected raw fenced tool_calls snippet stripped from content, got=%q", got) } - if streamFinishReason(frames) != "stop" { - t.Fatalf("expected finish_reason=stop, body=%s", rec.Body.String()) + if streamFinishReason(frames) != "tool_calls" { + t.Fatalf("expected finish_reason=tool_calls, body=%s", rec.Body.String()) } } diff --git a/internal/adapter/openai/tool_sieve_core.go b/internal/adapter/openai/tool_sieve_core.go index 72628e9..ca2223a 100644 --- a/internal/adapter/openai/tool_sieve_core.go +++ b/internal/adapter/openai/tool_sieve_core.go @@ -168,36 +168,21 @@ func findToolSegmentStart(s string) int { } lower := strings.ToLower(s) keywords := []string{"tool_calls", "function.name:", "[tool_call_history]"} - offset := 0 - for { - bestKeyIdx := -1 - matchedKeyword := "" - - for _, kw := range keywords { - idx := strings.Index(lower[offset:], kw) - if idx >= 0 { - absIdx := offset + idx - if bestKeyIdx < 0 || absIdx < bestKeyIdx { - bestKeyIdx = absIdx - matchedKeyword = kw - } - } + bestKeyIdx := -1 + for _, kw := range keywords { + idx := strings.Index(lower, kw) + if idx >= 0 && (bestKeyIdx < 0 || idx < bestKeyIdx) { + bestKeyIdx = idx } - - if bestKeyIdx < 0 { - return -1 - } - - keyIdx := bestKeyIdx - start := strings.LastIndex(s[:keyIdx], "{") - if start < 0 { - start = keyIdx - } - if !insideCodeFence(s[:start]) { - return start - } - offset = keyIdx + len(matchedKeyword) } + if bestKeyIdx < 0 { + return -1 + } + start := strings.LastIndex(s[:bestKeyIdx], "{") + if start < 0 { + start = bestKeyIdx + } + return start } func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix string, calls []util.ParsedToolCall, suffix string, ready bool) { @@ -229,9 +214,6 @@ func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix } prefixPart := captured[:start] suffixPart := captured[end:] - if insideCodeFence(state.recentTextTail + prefixPart) { - return captured, nil, "", true - } parsed := util.ParseStandaloneToolCallsDetailed(obj, toolNames) if len(parsed.Calls) == 0 { if parsed.SawToolCallSyntax && parsed.RejectedByPolicy { diff --git a/internal/adapter/openai/tool_sieve_incremental.go b/internal/adapter/openai/tool_sieve_incremental.go index ad0f901..d0d7842 100644 --- a/internal/adapter/openai/tool_sieve_incremental.go +++ b/internal/adapter/openai/tool_sieve_incremental.go @@ -19,9 +19,6 @@ func buildIncrementalToolDeltas(state *toolStreamSieveState) []toolCallDelta { if start < 0 { return nil } - if insideCodeFence(state.recentTextTail + captured[:start]) { - return nil - } certainSingle, hasMultiple := classifyToolCallsIncrementalSafety(captured, keyIdx) if hasMultiple { state.disableDeltas = true diff --git a/internal/format/openai/render_test.go b/internal/format/openai/render_test.go index 2ec05c6..952d0ef 100644 --- a/internal/format/openai/render_test.go +++ b/internal/format/openai/render_test.go @@ -69,7 +69,7 @@ func TestBuildResponseObjectPromotesMixedProseToolPayloadToFunctionCall(t *testi } } -func TestBuildResponseObjectFencedToolPayloadRemainsText(t *testing.T) { +func TestBuildResponseObjectPromotesFencedToolPayloadToFunctionCall(t *testing.T) { obj := BuildResponseObject( "resp_test", "gpt-4o", @@ -80,16 +80,16 @@ func TestBuildResponseObjectFencedToolPayloadRemainsText(t *testing.T) { ) outputText, _ := obj["output_text"].(string) - if outputText == "" { - t.Fatalf("expected output_text preserved for fenced example") + if outputText != "" { + t.Fatalf("expected output_text hidden for fenced tool payload, got %q", outputText) } output, _ := obj["output"].([]any) if len(output) != 1 { - t.Fatalf("expected one message output item, got %#v", obj["output"]) + t.Fatalf("expected one function_call output item, got %#v", obj["output"]) } first, _ := output[0].(map[string]any) - if first["type"] != "message" { - t.Fatalf("expected message output type, got %#v", first["type"]) + if first["type"] != "function_call" { + t.Fatalf("expected function_call output type, got %#v", first["type"]) } } diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index e55861b..bcffd03 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -26,10 +26,6 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa if strings.TrimSpace(text) == "" { return result } - text = stripFencedCodeBlocks(text) - if strings.TrimSpace(text) == "" { - return result - } result.SawToolCallSyntax = looksLikeToolCallSyntax(text) candidates := buildToolCallCandidates(text) @@ -75,7 +71,7 @@ func ParseStandaloneToolCalls(text string, availableToolNames []string) []Parsed func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) ToolCallParseResult { result := ToolCallParseResult{} - trimmed := strings.TrimSpace(stripFencedCodeBlocks(text)) + trimmed := strings.TrimSpace(text) if trimmed == "" { return result } diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index da6e59a..2d29c1a 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -22,8 +22,8 @@ func TestParseToolCalls(t *testing.T) { func TestParseToolCallsFromFencedJSON(t *testing.T) { text := "I will call tools now\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"news\"}}]}\n```" calls := ParseToolCalls(text, []string{"search"}) - if len(calls) != 0 { - t.Fatalf("expected fenced tool_call example to be ignored, got %#v", calls) + if len(calls) != 1 { + t.Fatalf("expected fenced tool_call payload to be parsed, got %#v", calls) } } @@ -112,10 +112,10 @@ func TestParseStandaloneToolCallsSupportsMixedProsePayload(t *testing.T) { } } -func TestParseStandaloneToolCallsIgnoresFencedCodeBlock(t *testing.T) { +func TestParseStandaloneToolCallsParsesFencedCodeBlock(t *testing.T) { fenced := "```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```" - if calls := ParseStandaloneToolCalls(fenced, []string{"search"}); len(calls) != 0 { - t.Fatalf("expected fenced tool_call example to be ignored, got %#v", calls) + if calls := ParseStandaloneToolCalls(fenced, []string{"search"}); len(calls) != 1 { + t.Fatalf("expected fenced tool_call payload to be parsed, got %#v", calls) } } diff --git a/internal/util/util_edge_test.go b/internal/util/util_edge_test.go index 876cd04..81d607e 100644 --- a/internal/util/util_edge_test.go +++ b/internal/util/util_edge_test.go @@ -409,8 +409,8 @@ func TestParseToolCallsWithFunctionWrapper(t *testing.T) { func TestParseStandaloneToolCallsFencedCodeBlock(t *testing.T) { fenced := "Here's an example:\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```\nDon't execute this." calls := ParseStandaloneToolCalls(fenced, []string{"search"}) - if len(calls) != 0 { - t.Fatalf("expected fenced code block ignored, got %d calls", len(calls)) + if len(calls) != 1 { + t.Fatalf("expected fenced code block to be parsed, got %d calls", len(calls)) } } diff --git a/tests/compat/expected/toolcalls_fenced_json.json b/tests/compat/expected/toolcalls_fenced_json.json index d740e67..124de59 100644 --- a/tests/compat/expected/toolcalls_fenced_json.json +++ b/tests/compat/expected/toolcalls_fenced_json.json @@ -1,6 +1,13 @@ { - "calls": [], - "sawToolCallSyntax": false, + "calls": [ + { + "name": "read_file", + "input": { + "path": "README.MD" + } + } + ], + "sawToolCallSyntax": true, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} diff --git a/tests/compat/expected/toolcalls_standalone_fenced_example.json b/tests/compat/expected/toolcalls_standalone_fenced_example.json index d740e67..124de59 100644 --- a/tests/compat/expected/toolcalls_standalone_fenced_example.json +++ b/tests/compat/expected/toolcalls_standalone_fenced_example.json @@ -1,6 +1,13 @@ { - "calls": [], - "sawToolCallSyntax": false, + "calls": [ + { + "name": "read_file", + "input": { + "path": "README.MD" + } + } + ], + "sawToolCallSyntax": true, "rejectedByPolicy": false, "rejectedToolNames": [] -} \ No newline at end of file +} From c4ec14f49ae2d3b2b60ad0759bae8b67131c652f Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:12:34 +0800 Subject: [PATCH 08/12] Fix refactor line gate for toolcalls_parse --- internal/util/toolcalls_parse.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index bcffd03..3880c2e 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -16,7 +16,6 @@ type ToolCallParseResult struct { RejectedByPolicy bool RejectedToolNames []string } - func ParseToolCalls(text string, availableToolNames []string) []ParsedToolCall { return ParseToolCallsDetailed(text, availableToolNames).Calls } @@ -64,7 +63,6 @@ func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallPa result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0 return result } - func ParseStandaloneToolCalls(text string, availableToolNames []string) []ParsedToolCall { return ParseStandaloneToolCallsDetailed(text, availableToolNames).Calls } From 184a3d1e4eb0d53ce8c01c56842b5530a106ac53 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:16:37 +0800 Subject: [PATCH 09/12] Sync Node tool-call parsing with aggressive fenced/mixed policy --- .../js/helpers/stream-tool-sieve/parse.js | 50 ++++++++++--------- tests/node/stream-tool-sieve.test.js | 12 +++-- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/internal/js/helpers/stream-tool-sieve/parse.js b/internal/js/helpers/stream-tool-sieve/parse.js index 6e6ff7d..22d11d1 100644 --- a/internal/js/helpers/stream-tool-sieve/parse.js +++ b/internal/js/helpers/stream-tool-sieve/parse.js @@ -2,10 +2,8 @@ const { toStringSafe, - looksLikeToolExampleContext, } = require('./state'); const { - stripFencedCodeBlocks, buildToolCallCandidates, parseToolCallsPayload, parseMarkupToolCalls, @@ -38,16 +36,13 @@ function parseToolCalls(text, toolNames) { function parseToolCallsDetailed(text, toolNames) { const result = emptyParseResult(); - if (!toStringSafe(text)) { + const normalized = toStringSafe(text); + if (!normalized) { return result; } - const sanitized = stripFencedCodeBlocks(text); - if (!toStringSafe(sanitized)) { - return result; - } - result.sawToolCallSyntax = looksLikeToolCallSyntax(sanitized); + result.sawToolCallSyntax = looksLikeToolCallSyntax(normalized); - const candidates = buildToolCallCandidates(sanitized); + const candidates = buildToolCallCandidates(normalized); let parsed = []; for (const c of candidates) { parsed = parseToolCallsPayload(c); @@ -63,9 +58,9 @@ function parseToolCallsDetailed(text, toolNames) { } } if (parsed.length === 0) { - parsed = parseMarkupToolCalls(sanitized); + parsed = parseMarkupToolCalls(normalized); if (parsed.length === 0) { - parsed = parseTextKVToolCalls(sanitized); + parsed = parseTextKVToolCalls(normalized); if (parsed.length === 0) { return result; } @@ -90,22 +85,29 @@ function parseStandaloneToolCallsDetailed(text, toolNames) { if (!trimmed) { return result; } - if (trimmed.includes('```')) { - return result; - } - if (looksLikeToolExampleContext(trimmed)) { - return result; - } result.sawToolCallSyntax = looksLikeToolCallSyntax(trimmed); - let parsed = parseToolCallsPayload(trimmed); + const candidates = buildToolCallCandidates(trimmed); + let parsed = []; + for (const c of candidates) { + parsed = parseToolCallsPayload(c); + if (parsed.length === 0) { + parsed = parseMarkupToolCalls(c); + } + if (parsed.length === 0) { + parsed = parseTextKVToolCalls(c); + } + if (parsed.length > 0) { + break; + } + } if (parsed.length === 0) { parsed = parseMarkupToolCalls(trimmed); - } - if (parsed.length === 0) { - parsed = parseTextKVToolCalls(trimmed); - } - if (parsed.length === 0) { - return result; + if (parsed.length === 0) { + parsed = parseTextKVToolCalls(trimmed); + if (parsed.length === 0) { + return result; + } + } } result.sawToolCallSyntax = true; diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index 7c703a1..d4b5481 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -91,7 +91,9 @@ test('parseToolCalls supports fenced json and function.arguments string payload' '```', ].join('\n'); const calls = parseToolCalls(text, ['read_file']); - assert.equal(calls.length, 0); + assert.equal(calls.length, 1); + assert.equal(calls[0].name, 'read_file'); + assert.equal(calls[0].input.path, 'README.md'); }); test('parseToolCalls parses text-kv fallback payload', () => { @@ -122,19 +124,19 @@ test('parseToolCalls parses multiple text-kv fallback payloads', () => { assert.equal(calls[1].name, 'bash'); }); -test('parseStandaloneToolCalls only matches standalone payload and ignores mixed prose', () => { +test('parseStandaloneToolCalls parses mixed prose payload', () => { const mixed = '这里是示例：{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}，请勿执行。'; const standalone = '{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}'; const mixedCalls = parseStandaloneToolCalls(mixed, ['read_file']); const standaloneCalls = parseStandaloneToolCalls(standalone, ['read_file']); - assert.equal(mixedCalls.length, 0); + assert.equal(mixedCalls.length, 1); assert.equal(standaloneCalls.length, 1); }); -test('parseStandaloneToolCalls ignores fenced code block tool_call examples', () => { +test('parseStandaloneToolCalls parses fenced code block tool_call payload', () => { const fenced = ['```json', '{"tool_calls":[{"name":"read_file","input":{"path":"README.MD"}}]}', '```'].join('\n'); const calls = parseStandaloneToolCalls(fenced, ['read_file']); - assert.equal(calls.length, 0); + assert.equal(calls.length, 1); }); From 99a61640004b39ef5e9e722c6a952ee70d0c8bb4 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:31:37 +0800 Subject: [PATCH 10/12] Fix path corruption when parsing tool call JSON strings --- internal/util/toolcalls_parse.go | 57 ++++++++++++++++++++++++++++++++ internal/util/toolcalls_test.go | 21 ++++++++++++ 2 files changed, 78 insertions(+) diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 3880c2e..08749a2 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -3,6 +3,7 @@ package util import ( "encoding/json" "strings" + "unicode" ) type ParsedToolCall struct { @@ -269,6 +270,7 @@ func parseToolCallInput(v any) map[string]any { } var parsed map[string]any if err := json.Unmarshal([]byte(raw), &parsed); err == nil && parsed != nil { + repairPathLikeControlChars(parsed) return parsed } // Try to repair invalid backslashes (common in Windows paths output by models) @@ -298,3 +300,58 @@ func parseToolCallInput(v any) map[string]any { return map[string]any{} } } + +func repairPathLikeControlChars(m map[string]any) { + for k, v := range m { + switch vv := v.(type) { + case map[string]any: + repairPathLikeControlChars(vv) + case []any: + for _, item := range vv { + if child, ok := item.(map[string]any); ok { + repairPathLikeControlChars(child) + } + } + case string: + if isPathLikeKey(k) && containsControlRune(vv) { + m[k] = escapeControlRunes(vv) + } + } + } +} + +func isPathLikeKey(key string) bool { + k := strings.ToLower(strings.TrimSpace(key)) + return strings.Contains(k, "path") || strings.Contains(k, "file") +} + +func containsControlRune(s string) bool { + for _, r := range s { + if unicode.IsControl(r) { + return true + } + } + return false +} + +func escapeControlRunes(s string) string { + var b strings.Builder + b.Grow(len(s) + 8) + for _, r := range s { + switch r { + case '\b': + b.WriteString(`\b`) + case '\f': + b.WriteString(`\f`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + b.WriteRune(r) + } + } + return b.String() +} diff --git a/internal/util/toolcalls_test.go b/internal/util/toolcalls_test.go index 2d29c1a..9701a46 100644 --- a/internal/util/toolcalls_test.go +++ b/internal/util/toolcalls_test.go @@ -419,6 +419,27 @@ func TestParseToolCallsWithMixedWindowsPaths(t *testing.T) { } } +func TestParseToolCallInputRepairsControlCharsInPath(t *testing.T) { + in := `{"path":"D:\tmp\new\readme.txt","content":"line1\nline2"}` + parsed := parseToolCallInput(in) + + path, ok := parsed["path"].(string) + if !ok { + t.Fatalf("expected path string in parsed input, got %#v", parsed["path"]) + } + if path != `D:\tmp\new\readme.txt` { + t.Fatalf("expected repaired windows path, got %q", path) + } + + content, ok := parsed["content"].(string) + if !ok { + t.Fatalf("expected content string in parsed input, got %#v", parsed["content"]) + } + if content != "line1\nline2" { + t.Fatalf("expected non-path field to keep decoded escapes, got %q", content) + } +} + func TestRepairLooseJSONWithNestedObjects(t *testing.T) { // 测试嵌套对象的修复：DeepSeek 幻觉输出，每个元素内部包含嵌套 {} // 注意：正则只支持单层嵌套，不支持更深层次的嵌套 From acfb3b225de7faa434a5d004bfb018266ec249b4 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:37:23 +0800 Subject: [PATCH 11/12] Split toolcall input parsing to satisfy line gate --- internal/util/toolcalls_input_parse.go | 106 +++++++++++++++++++++++++ internal/util/toolcalls_parse.go | 100 ----------------------- 2 files changed, 106 insertions(+), 100 deletions(-) create mode 100644 internal/util/toolcalls_input_parse.go diff --git a/internal/util/toolcalls_input_parse.go b/internal/util/toolcalls_input_parse.go new file mode 100644 index 0000000..14429c0 --- /dev/null +++ b/internal/util/toolcalls_input_parse.go @@ -0,0 +1,106 @@ +package util + +import ( + "encoding/json" + "strings" + "unicode" +) + +func parseToolCallInput(v any) map[string]any { + switch x := v.(type) { + case nil: + return map[string]any{} + case map[string]any: + return x + case string: + raw := strings.TrimSpace(x) + if raw == "" { + return map[string]any{} + } + var parsed map[string]any + if err := json.Unmarshal([]byte(raw), &parsed); err == nil && parsed != nil { + repairPathLikeControlChars(parsed) + return parsed + } + // Try to repair invalid backslashes (common in Windows paths output by models) + repaired := repairInvalidJSONBackslashes(raw) + if repaired != raw { + if err := json.Unmarshal([]byte(repaired), &parsed); err == nil && parsed != nil { + return parsed + } + } + // Try to repair loose JSON in string argument as well + repairedLoose := RepairLooseJSON(raw) + if repairedLoose != raw { + if err := json.Unmarshal([]byte(repairedLoose), &parsed); err == nil && parsed != nil { + return parsed + } + } + return map[string]any{"_raw": raw} + default: + b, err := json.Marshal(x) + if err != nil { + return map[string]any{} + } + var parsed map[string]any + if err := json.Unmarshal(b, &parsed); err == nil && parsed != nil { + return parsed + } + return map[string]any{} + } +} + +func repairPathLikeControlChars(m map[string]any) { + for k, v := range m { + switch vv := v.(type) { + case map[string]any: + repairPathLikeControlChars(vv) + case []any: + for _, item := range vv { + if child, ok := item.(map[string]any); ok { + repairPathLikeControlChars(child) + } + } + case string: + if isPathLikeKey(k) && containsControlRune(vv) { + m[k] = escapeControlRunes(vv) + } + } + } +} + +func isPathLikeKey(key string) bool { + k := strings.ToLower(strings.TrimSpace(key)) + return strings.Contains(k, "path") || strings.Contains(k, "file") +} + +func containsControlRune(s string) bool { + for _, r := range s { + if unicode.IsControl(r) { + return true + } + } + return false +} + +func escapeControlRunes(s string) string { + var b strings.Builder + b.Grow(len(s) + 8) + for _, r := range s { + switch r { + case '\b': + b.WriteString(`\b`) + case '\f': + b.WriteString(`\f`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + b.WriteRune(r) + } + } + return b.String() +} diff --git a/internal/util/toolcalls_parse.go b/internal/util/toolcalls_parse.go index 08749a2..aa2288a 100644 --- a/internal/util/toolcalls_parse.go +++ b/internal/util/toolcalls_parse.go @@ -3,7 +3,6 @@ package util import ( "encoding/json" "strings" - "unicode" ) type ParsedToolCall struct { @@ -256,102 +255,3 @@ func parseToolCallItem(m map[string]any) (ParsedToolCall, bool) { Input: parseToolCallInput(inputRaw), }, true } - -func parseToolCallInput(v any) map[string]any { - switch x := v.(type) { - case nil: - return map[string]any{} - case map[string]any: - return x - case string: - raw := strings.TrimSpace(x) - if raw == "" { - return map[string]any{} - } - var parsed map[string]any - if err := json.Unmarshal([]byte(raw), &parsed); err == nil && parsed != nil { - repairPathLikeControlChars(parsed) - return parsed - } - // Try to repair invalid backslashes (common in Windows paths output by models) - repaired := repairInvalidJSONBackslashes(raw) - if repaired != raw { - if err := json.Unmarshal([]byte(repaired), &parsed); err == nil && parsed != nil { - return parsed - } - } - // Try to repair loose JSON in string argument as well - repairedLoose := RepairLooseJSON(raw) - if repairedLoose != raw { - if err := json.Unmarshal([]byte(repairedLoose), &parsed); err == nil && parsed != nil { - return parsed - } - } - return map[string]any{"_raw": raw} - default: - b, err := json.Marshal(x) - if err != nil { - return map[string]any{} - } - var parsed map[string]any - if err := json.Unmarshal(b, &parsed); err == nil && parsed != nil { - return parsed - } - return map[string]any{} - } -} - -func repairPathLikeControlChars(m map[string]any) { - for k, v := range m { - switch vv := v.(type) { - case map[string]any: - repairPathLikeControlChars(vv) - case []any: - for _, item := range vv { - if child, ok := item.(map[string]any); ok { - repairPathLikeControlChars(child) - } - } - case string: - if isPathLikeKey(k) && containsControlRune(vv) { - m[k] = escapeControlRunes(vv) - } - } - } -} - -func isPathLikeKey(key string) bool { - k := strings.ToLower(strings.TrimSpace(key)) - return strings.Contains(k, "path") || strings.Contains(k, "file") -} - -func containsControlRune(s string) bool { - for _, r := range s { - if unicode.IsControl(r) { - return true - } - } - return false -} - -func escapeControlRunes(s string) string { - var b strings.Builder - b.Grow(len(s) + 8) - for _, r := range s { - switch r { - case '\b': - b.WriteString(`\b`) - case '\f': - b.WriteString(`\f`) - case '\n': - b.WriteString(`\n`) - case '\r': - b.WriteString(`\r`) - case '\t': - b.WriteString(`\t`) - default: - b.WriteRune(r) - } - } - return b.String() -} From 88a79f212d903afe6adcc97593b8e05eb646d816 Mon Sep 17 00:00:00 2001 From: "CJACK." Date: Fri, 20 Mar 2026 02:52:27 +0800 Subject: [PATCH 12/12] Fix path control-char repair on JSON fallback parses --- internal/util/toolcalls_input_parse.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/util/toolcalls_input_parse.go b/internal/util/toolcalls_input_parse.go index 14429c0..4f08474 100644 --- a/internal/util/toolcalls_input_parse.go +++ b/internal/util/toolcalls_input_parse.go @@ -26,6 +26,7 @@ func parseToolCallInput(v any) map[string]any { repaired := repairInvalidJSONBackslashes(raw) if repaired != raw { if err := json.Unmarshal([]byte(repaired), &parsed); err == nil && parsed != nil { + repairPathLikeControlChars(parsed) return parsed } } @@ -33,6 +34,7 @@ func parseToolCallInput(v any) map[string]any { repairedLoose := RepairLooseJSON(raw) if repairedLoose != raw { if err := json.Unmarshal([]byte(repairedLoose), &parsed); err == nil && parsed != nil { + repairPathLikeControlChars(parsed) return parsed } }