refactor: move tool call parsing and formatting logic to a dedicated internal/toolcall package

This commit is contained in:
CJACK
2026-04-06 03:19:18 +08:00
parent 2857a171cc
commit 1530246e4f
39 changed files with 261 additions and 159 deletions

View File

@@ -21,6 +21,12 @@ jobs:
with: with:
go-version: "1.26.x" go-version: "1.26.x"
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
version: latest
args: --timeout=5m
- name: Setup Node - name: Setup Node
uses: actions/setup-node@v4 uses: actions/setup-node@v4
with: with:

80
.golangci.yml Normal file
View File

@@ -0,0 +1,80 @@
linters-settings:
govet:
check-shadowing: true
golint:
min-confidence: 0
gocyclo:
min-complexity: 15
maligned:
suggest-new: true
dupl:
threshold: 100
goconst:
min-len: 2
min-occurrences: 2
misspell:
locale: US
lll:
line-length: 140
goimports:
local-prefixes: ds2api
unused:
check-exported: false
unparam:
check-exported: false
nakedret:
max-func-lines: 30
prealloc:
simple: true
range-loops: true
for-loops: false
gocritic:
enabled-tags:
- diagnostic
- experimental
- opinionated
- performance
- style
disabled-checks:
- wrapperFunc
- rangeValCopy
- hugeParam
linters:
enable:
- govet
- errcheck
- staticcheck
- unused
- gosimple
- structcheck
- varcheck
- ineffassign
- deadcode
- typecheck
- bodyclose
- stylecheck
- revive
- unconvert
- goconst
- gocyclo
- asciicheck
- gofmt
- misspell
- nakedret
- exportloopref
- dogsled
issues:
exclude-use-default: false
max-issues-per-linter: 0
max-same-issues: 0
exclude:
- "ST1000: at least one file in a package should have a package comment"
run:
timeout: 5m
tests: true
skip-dirs:
- vendor
- webui/node_modules

View File

@@ -81,7 +81,7 @@ flowchart LR
- **统一路由内核**:所有协议入口统一汇聚到 `internal/server/router.go`,并在同一路由树中注册 OpenAI / Claude / Gemini / Admin / WebUI 路由,避免多入口行为漂移。 - **统一路由内核**:所有协议入口统一汇聚到 `internal/server/router.go`,并在同一路由树中注册 OpenAI / Claude / Gemini / Admin / WebUI 路由,避免多入口行为漂移。
- **统一执行链路**Claude / Gemini 入口先经 `internal/translatorcliproxy` 做协议转换,再进入 `openai.ChatCompletions` 统一处理工具调用与流式语义,最后再转换回原协议响应。 - **统一执行链路**Claude / Gemini 入口先经 `internal/translatorcliproxy` 做协议转换,再进入 `openai.ChatCompletions` 统一处理工具调用与流式语义,最后再转换回原协议响应。
- **适配器分层更清晰**`internal/adapter/{claude,gemini}` 负责入口/出口协议封装,`internal/adapter/openai` 负责核心执行DeepSeek 侧调用只保留在 OpenAI 内核中。 - **适配器分层更清晰**`internal/adapter/{claude,gemini}` 负责入口/出口协议封装,`internal/adapter/openai` 负责核心执行DeepSeek 侧调用只保留在 OpenAI 内核中。
- **Tool Calling 双运行时对齐**Go 侧(`internal/util`)与 Vercel Node 侧(`internal/js/helpers/stream-tool-sieve`)保持一致的解析/防泄漏语义,覆盖 JSON / XML / invoke / text-kv 多风格输入。 - **Tool Calling 双运行时对齐**Go 侧(`internal/toolcall`)与 Vercel Node 侧(`internal/js/helpers/stream-tool-sieve`)保持一致的解析/防泄漏语义,覆盖 JSON / XML / invoke / text-kv 多风格输入。
- **配置与运行时设置解耦**:静态配置(`config`)与运行时策略(`settings`)通过 Admin API 分离管理,支持热更新和密码轮换失效旧 JWT。 - **配置与运行时设置解耦**:静态配置(`config`)与运行时策略(`settings`)通过 Admin API 分离管理,支持热更新和密码轮换失效旧 JWT。
- **流式能力升级**`/v1/responses` 与 `/v1/chat/completions` 共享更一致的工具调用增量输出策略,降低不同 SDK 下的行为差异。 - **流式能力升级**`/v1/responses` 与 `/v1/chat/completions` 共享更一致的工具调用增量输出策略,降低不同 SDK 下的行为差异。
- **可观测与可运维增强**`/healthz`、`/readyz`、`/admin/version`、`/admin/dev/captures` 形成排障闭环,便于发布后验证。 - **可观测与可运维增强**`/healthz`、`/readyz`、`/admin/version`、`/admin/dev/captures` 形成排障闭环,便于发布后验证。
@@ -466,7 +466,8 @@ ds2api/
│ ├── stream/ # 统一流式消费引擎 │ ├── stream/ # 统一流式消费引擎
│ ├── testsuite/ # 端到端测试框架与用例编排 │ ├── testsuite/ # 端到端测试框架与用例编排
│ ├── translatorcliproxy/ # CLIProxy 桥接与流写入组件 │ ├── translatorcliproxy/ # CLIProxy 桥接与流写入组件
│ ├── util/ # 通用工具函数 │ ├── toolcall/ # Tool Call 解析、修复与格式化(核心业务逻辑)
│ ├── util/ # 通用工具函数Token 估算、JSON 辅助等)
│ ├── version/ # 版本解析 / 比较与 tag 规范化 │ ├── version/ # 版本解析 / 比较与 tag 规范化
│ └── webui/ # WebUI 静态文件托管与自动构建 │ └── webui/ # WebUI 静态文件托管与自动构建
├── webui/ # React WebUI 源码Vite + Tailwind ├── webui/ # React WebUI 源码Vite + Tailwind
@@ -543,7 +544,7 @@ npm ci --prefix webui && npm run build --prefix webui
go test ./... go test ./...
# 运行 tool calls 相关测试(调试工具调用问题) # 运行 tool calls 相关测试(调试工具调用问题)
go test -v -run 'TestParseToolCalls|TestRepair' ./internal/util/ go test -v -run 'TestParseToolCalls|TestRepair' ./internal/toolcall/
# 运行端到端测试 # 运行端到端测试
./tests/scripts/run-live.sh ./tests/scripts/run-live.sh

View File

@@ -81,7 +81,7 @@ flowchart LR
- **Unified routing core**: all protocol entries are now centralized through `internal/server/router.go`, with OpenAI / Claude / Gemini / Admin / WebUI routes registered in one tree to avoid multi-entry drift. - **Unified routing core**: all protocol entries are now centralized through `internal/server/router.go`, with OpenAI / Claude / Gemini / Admin / WebUI routes registered in one tree to avoid multi-entry drift.
- **Unified execution chain**: Claude/Gemini entries are translated by `internal/translatorcliproxy`, then executed through `openai.ChatCompletions` for shared tool-calling and stream semantics, then translated back to the client protocol. - **Unified execution chain**: Claude/Gemini entries are translated by `internal/translatorcliproxy`, then executed through `openai.ChatCompletions` for shared tool-calling and stream semantics, then translated back to the client protocol.
- **Cleaner adapter boundaries**: `internal/adapter/{claude,gemini}` handles protocol wrappers, while `internal/adapter/openai` remains the execution core; upstream DeepSeek calls are retained only in the OpenAI core. - **Cleaner adapter boundaries**: `internal/adapter/{claude,gemini}` handles protocol wrappers, while `internal/adapter/openai` remains the execution core; upstream DeepSeek calls are retained only in the OpenAI core.
- **Tool-calling parity across runtimes**: Go (`internal/util`) and Vercel Node (`internal/js/helpers/stream-tool-sieve`) follow aligned parsing/anti-leak semantics across JSON / XML / invoke / text-kv inputs. - **Tool-calling parity across runtimes**: Go (`internal/toolcall`) and Vercel Node (`internal/js/helpers/stream-tool-sieve`) follow aligned parsing/anti-leak semantics across JSON / XML / invoke / text-kv inputs.
- **Config/runtime separation**: static config (`config`) and runtime policy (`settings`) are managed independently via Admin APIs, enabling hot updates and password rotation with JWT invalidation. - **Config/runtime separation**: static config (`config`) and runtime policy (`settings`) are managed independently via Admin APIs, enabling hot updates and password rotation with JWT invalidation.
- **Streaming behavior upgrade**: `/v1/responses` and `/v1/chat/completions` now share a more consistent incremental tool-call emission strategy across SDK ecosystems. - **Streaming behavior upgrade**: `/v1/responses` and `/v1/chat/completions` now share a more consistent incremental tool-call emission strategy across SDK ecosystems.
- **Improved operability**: `/healthz`, `/readyz`, `/admin/version`, and `/admin/dev/captures` form a tighter post-deploy diagnostics loop. - **Improved operability**: `/healthz`, `/readyz`, `/admin/version`, and `/admin/dev/captures` form a tighter post-deploy diagnostics loop.
@@ -464,7 +464,8 @@ ds2api/
│ ├── stream/ # Unified stream consumption engine │ ├── stream/ # Unified stream consumption engine
│ ├── testsuite/ # End-to-end testsuite framework and case orchestration │ ├── testsuite/ # End-to-end testsuite framework and case orchestration
│ ├── translatorcliproxy/ # CLIProxy bridge and stream writer components │ ├── translatorcliproxy/ # CLIProxy bridge and stream writer components
│ ├── util/ # Common utilities │ ├── toolcall/ # Tool Call parsing, repair, and formatting (core business logic)
│ ├── util/ # Common utilities (Token estimation, JSON helpers, etc.)
│ ├── version/ # Version parsing/comparison and tag normalization │ ├── version/ # Version parsing/comparison and tag normalization
│ └── webui/ # WebUI static file serving and auto-build │ └── webui/ # WebUI static file serving and auto-build
├── webui/ # React WebUI source (Vite + Tailwind) ├── webui/ # React WebUI source (Vite + Tailwind)

View File

@@ -1 +1 @@
3.1.1 3.1.2

View File

@@ -180,10 +180,10 @@ go test ./...
```bash ```bash
# 运行 tool calls 相关测试(推荐用于调试 tool call 解析问题) # 运行 tool calls 相关测试(推荐用于调试 tool call 解析问题)
go test -v -run 'TestParseToolCalls|TestRepair' ./internal/util/ go test -v -run 'TestParseToolCalls|TestRepair' ./internal/toolcall/
# 运行单个测试用例 # 运行单个测试用例
go test -v -run TestParseToolCallsWithDeepSeekHallucination ./internal/util/ go test -v -run TestParseToolCallsWithDeepSeekHallucination ./internal/toolcall/
# 运行 format 相关测试 # 运行 format 相关测试
go test -v ./internal/format/... go test -v ./internal/format/...
@@ -198,13 +198,13 @@ go test -v ./internal/adapter/openai/...
```bash ```bash
# 1. 运行 tool calls 相关的所有测试 # 1. 运行 tool calls 相关的所有测试
go test -v -run 'TestParseToolCalls|TestRepair' ./internal/util/ go test -v -run 'TestParseToolCalls|TestRepair' ./internal/toolcall/
# 2. 查看测试输出中的详细调试信息 # 2. 查看测试输出中的详细调试信息
go test -v -run TestParseToolCallsWithDeepSeekHallucination ./internal/util/ 2>&1 go test -v -run TestParseToolCallsWithDeepSeekHallucination ./internal/toolcall/ 2>&1
# 3. 检查具体测试用例的修复效果 # 3. 检查具体测试用例的修复效果
# 测试用例位于 internal/util/toolcalls_test.go包含 # 测试用例位于 internal/toolcall/toolcalls_test.go包含
# - TestParseToolCallsWithDeepSeekHallucination: DeepSeek 典型幻觉输出 # - TestParseToolCallsWithDeepSeekHallucination: DeepSeek 典型幻觉输出
# - TestRepairLooseJSONWithNestedObjects: 嵌套对象的方括号修复 # - TestRepairLooseJSONWithNestedObjects: 嵌套对象的方括号修复
# - TestParseToolCallsWithMixedWindowsPaths: Windows 路径处理 # - TestParseToolCallsWithMixedWindowsPaths: Windows 路径处理

View File

@@ -1,12 +1,12 @@
package claude package claude
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings" "strings"
"ds2api/internal/prompt" "ds2api/internal/prompt"
"ds2api/internal/util"
) )
func normalizeClaudeMessages(messages []any) []any { func normalizeClaudeMessages(messages []any) []any {
@@ -98,7 +98,7 @@ func buildClaudeToolPrompt(tools []any) string {
} }
return "You have access to these tools:\n\n" + return "You have access to these tools:\n\n" +
strings.Join(toolSchemas, "\n\n") + "\n\n" + strings.Join(toolSchemas, "\n\n") + "\n\n" +
util.BuildToolCallInstructions(names) toolcall.BuildToolCallInstructions(names)
} }
func formatClaudeToolResultForPrompt(block map[string]any) string { func formatClaudeToolResultForPrompt(block map[string]any) string {

View File

@@ -1,6 +1,7 @@
package claude package claude
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"fmt" "fmt"
"time" "time"
@@ -46,9 +47,9 @@ func (s *claudeStreamRuntime) finalize(stopReason string) {
finalText := cleanVisibleOutput(s.text.String(), s.stripReferenceMarkers) finalText := cleanVisibleOutput(s.text.String(), s.stripReferenceMarkers)
if s.bufferToolContent { if s.bufferToolContent {
detected := util.ParseStandaloneToolCalls(finalText, s.toolNames) detected := toolcall.ParseStandaloneToolCalls(finalText, s.toolNames)
if len(detected) == 0 && finalText == "" && finalThinking != "" { if len(detected) == 0 && finalText == "" && finalThinking != "" {
detected = util.ParseStandaloneToolCalls(finalThinking, s.toolNames) detected = toolcall.ParseStandaloneToolCalls(finalThinking, s.toolNames)
} }
if len(detected) > 0 { if len(detected) > 0 {
stopReason = "tool_use" stopReason = "tool_use"

View File

@@ -1,6 +1,7 @@
package gemini package gemini
import ( import (
"ds2api/internal/toolcall"
"bytes" "bytes"
"encoding/json" "encoding/json"
"io" "io"
@@ -186,9 +187,9 @@ func buildGeminiUsage(finalPrompt, finalThinking, finalText string, outputTokens
} }
func buildGeminiPartsFromFinal(finalText, finalThinking string, toolNames []string) []map[string]any { func buildGeminiPartsFromFinal(finalText, finalThinking string, toolNames []string) []map[string]any {
detected := util.ParseToolCalls(finalText, toolNames) detected := toolcall.ParseToolCalls(finalText, toolNames)
if len(detected) == 0 && finalThinking != "" { if len(detected) == 0 && finalThinking != "" {
detected = util.ParseToolCalls(finalThinking, toolNames) detected = toolcall.ParseToolCalls(finalThinking, toolNames)
} }
if len(detected) > 0 { if len(detected) > 0 {
parts := make([]map[string]any, 0, len(detected)) parts := make([]map[string]any, 0, len(detected))

View File

@@ -1,6 +1,7 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"net/http" "net/http"
"strings" "strings"
@@ -8,7 +9,6 @@ import (
openaifmt "ds2api/internal/format/openai" openaifmt "ds2api/internal/format/openai"
"ds2api/internal/sse" "ds2api/internal/sse"
streamengine "ds2api/internal/stream" streamengine "ds2api/internal/stream"
"ds2api/internal/util"
) )
type chatStreamRuntime struct { type chatStreamRuntime struct {
@@ -102,7 +102,7 @@ func (s *chatStreamRuntime) sendDone() {
func (s *chatStreamRuntime) finalize(finishReason string) { func (s *chatStreamRuntime) finalize(finishReason string) {
finalThinking := s.thinking.String() finalThinking := s.thinking.String()
finalText := cleanVisibleOutput(s.text.String(), s.stripReferenceMarkers) finalText := cleanVisibleOutput(s.text.String(), s.stripReferenceMarkers)
detected := util.ParseStandaloneToolCallsDetailed(finalText, s.toolNames) detected := toolcall.ParseStandaloneToolCallsDetailed(finalText, s.toolNames)
if len(detected.Calls) > 0 && !s.toolCallsDoneEmitted { if len(detected.Calls) > 0 && !s.toolCallsDoneEmitted {
finishReason = "tool_calls" finishReason = "tool_calls"
delta := map[string]any{ delta := map[string]any{

View File

@@ -1,6 +1,7 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings" "strings"
@@ -75,7 +76,7 @@ func injectToolPrompt(messages []map[string]any, tools []any, policy util.ToolCh
// buildToolCallInstructions delegates to the shared util implementation. // buildToolCallInstructions delegates to the shared util implementation.
func buildToolCallInstructions(toolNames []string) string { func buildToolCallInstructions(toolNames []string) string {
return util.BuildToolCallInstructions(toolNames) return toolcall.BuildToolCallInstructions(toolNames)
} }
func formatIncrementalStreamToolCallDeltas(deltas []toolCallDelta, ids map[int]string) []map[string]any { func formatIncrementalStreamToolCallDeltas(deltas []toolCallDelta, ids map[int]string) []map[string]any {
@@ -138,7 +139,7 @@ func filterIncrementalToolCallDeltasByAllowed(deltas []toolCallDelta, allowedNam
return out return out
} }
func formatFinalStreamToolCallsWithStableIDs(calls []util.ParsedToolCall, ids map[int]string) []map[string]any { func formatFinalStreamToolCallsWithStableIDs(calls []toolcall.ParsedToolCall, ids map[int]string) []map[string]any {
if len(calls) == 0 { if len(calls) == 0 {
return nil return nil
} }

View File

@@ -1,6 +1,7 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"io" "io"
"net/http" "net/http"
@@ -119,7 +120,7 @@ func (h *Handler) handleResponsesNonStream(w http.ResponseWriter, resp *http.Res
if writeUpstreamEmptyOutputError(w, sanitizedThinking, sanitizedText, result.ContentFilter) { if writeUpstreamEmptyOutputError(w, sanitizedThinking, sanitizedText, result.ContentFilter) {
return return
} }
textParsed := util.ParseStandaloneToolCallsDetailed(sanitizedText, toolNames) textParsed := toolcall.ParseStandaloneToolCallsDetailed(sanitizedText, toolNames)
logResponsesToolPolicyRejection(traceID, toolChoice, textParsed, "text") logResponsesToolPolicyRejection(traceID, toolChoice, textParsed, "text")
callCount := len(textParsed.Calls) callCount := len(textParsed.Calls)
@@ -200,7 +201,7 @@ func (h *Handler) handleResponsesStream(w http.ResponseWriter, r *http.Request,
}) })
} }
func logResponsesToolPolicyRejection(traceID string, policy util.ToolChoicePolicy, parsed util.ToolCallParseResult, channel string) { func logResponsesToolPolicyRejection(traceID string, policy util.ToolChoicePolicy, parsed toolcall.ToolCallParseResult, channel string) {
rejected := filteredRejectedToolNamesForLog(parsed.RejectedToolNames) rejected := filteredRejectedToolNamesForLog(parsed.RejectedToolNames)
if !parsed.RejectedByPolicy || len(rejected) == 0 { if !parsed.RejectedByPolicy || len(rejected) == 0 {
return return

View File

@@ -1,6 +1,7 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"net/http" "net/http"
"strings" "strings"
@@ -107,7 +108,7 @@ func (s *responsesStreamRuntime) finalize() {
s.processToolStreamEvents(flushToolSieve(&s.sieve, s.toolNames), true) s.processToolStreamEvents(flushToolSieve(&s.sieve, s.toolNames), true)
} }
textParsed := util.ParseStandaloneToolCallsDetailed(finalText, s.toolNames) textParsed := toolcall.ParseStandaloneToolCallsDetailed(finalText, s.toolNames)
detected := textParsed.Calls detected := textParsed.Calls
s.logToolPolicyRejections(textParsed) s.logToolPolicyRejections(textParsed)
@@ -163,8 +164,8 @@ func (s *responsesStreamRuntime) finalize() {
s.sendDone() s.sendDone()
} }
func (s *responsesStreamRuntime) logToolPolicyRejections(textParsed util.ToolCallParseResult) { func (s *responsesStreamRuntime) logToolPolicyRejections(textParsed toolcall.ToolCallParseResult) {
logRejected := func(parsed util.ToolCallParseResult, channel string) { logRejected := func(parsed toolcall.ToolCallParseResult, channel string) {
rejected := filteredRejectedToolNamesForLog(parsed.RejectedToolNames) rejected := filteredRejectedToolNamesForLog(parsed.RejectedToolNames)
if !parsed.RejectedByPolicy || len(rejected) == 0 { if !parsed.RejectedByPolicy || len(rejected) == 0 {
return return

View File

@@ -1,11 +1,11 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"strings" "strings"
openaifmt "ds2api/internal/format/openai" openaifmt "ds2api/internal/format/openai"
"ds2api/internal/util"
"github.com/google/uuid" "github.com/google/uuid"
) )
@@ -208,7 +208,7 @@ func (s *responsesStreamRuntime) emitFunctionCallDeltaEvents(deltas []toolCallDe
} }
} }
func (s *responsesStreamRuntime) emitFunctionCallDoneEvents(calls []util.ParsedToolCall) { func (s *responsesStreamRuntime) emitFunctionCallDoneEvents(calls []toolcall.ParsedToolCall) {
for idx, tc := range calls { for idx, tc := range calls {
if strings.TrimSpace(tc.Name) == "" { if strings.TrimSpace(tc.Name) == "" {
continue continue

View File

@@ -1,12 +1,12 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"sort" "sort"
"strings" "strings"
openaifmt "ds2api/internal/format/openai" openaifmt "ds2api/internal/format/openai"
"ds2api/internal/util"
) )
func (s *responsesStreamRuntime) closeIncompleteFunctionItems() { func (s *responsesStreamRuntime) closeIncompleteFunctionItems() {
@@ -57,7 +57,7 @@ func (s *responsesStreamRuntime) closeIncompleteFunctionItems() {
} }
} }
func (s *responsesStreamRuntime) buildCompletedResponseObject(finalThinking, finalText string, calls []util.ParsedToolCall) map[string]any { func (s *responsesStreamRuntime) buildCompletedResponseObject(finalThinking, finalText string, calls []toolcall.ParsedToolCall) map[string]any {
type indexedItem struct { type indexedItem struct {
index int index int
item map[string]any item map[string]any

View File

@@ -3,7 +3,7 @@ package openai
import ( import (
"strings" "strings"
"ds2api/internal/util" "ds2api/internal/toolcall"
) )
func processToolSieveChunk(state *toolStreamSieveState, chunk string, toolNames []string) []toolStreamEvent { func processToolSieveChunk(state *toolStreamSieveState, chunk string, toolNames []string) []toolStreamEvent {
@@ -226,7 +226,7 @@ func findToolSegmentStart(s string) int {
return start return start
} }
func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix string, calls []util.ParsedToolCall, suffix string, ready bool) { func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix string, calls []toolcall.ParsedToolCall, suffix string, ready bool) {
captured := state.capture.String() captured := state.capture.String()
if captured == "" { if captured == "" {
return "", nil, "", false return "", nil, "", false
@@ -267,7 +267,7 @@ func consumeToolCapture(state *toolStreamSieveState, toolNames []string) (prefix
} }
prefixPart := captured[:start] prefixPart := captured[:start]
suffixPart := captured[end:] suffixPart := captured[end:]
parsed := util.ParseStandaloneToolCallsDetailed(obj, toolNames) parsed := toolcall.ParseStandaloneToolCallsDetailed(obj, toolNames)
if len(parsed.Calls) == 0 { if len(parsed.Calls) == 0 {
if parsed.SawToolCallSyntax && parsed.RejectedByPolicy { if parsed.SawToolCallSyntax && parsed.RejectedByPolicy {
// Parsed as tool-call payload but rejected by schema/policy: // Parsed as tool-call payload but rejected by schema/policy:

View File

@@ -1,9 +1,9 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"strings" "strings"
"ds2api/internal/util"
) )
type toolStreamSieveState struct { type toolStreamSieveState struct {
@@ -12,7 +12,7 @@ type toolStreamSieveState struct {
capturing bool capturing bool
recentTextTail string recentTextTail string
pendingToolRaw string pendingToolRaw string
pendingToolCalls []util.ParsedToolCall pendingToolCalls []toolcall.ParsedToolCall
disableDeltas bool disableDeltas bool
toolNameSent bool toolNameSent bool
toolName string toolName string
@@ -24,7 +24,7 @@ type toolStreamSieveState struct {
type toolStreamEvent struct { type toolStreamEvent struct {
Content string Content string
ToolCalls []util.ParsedToolCall ToolCalls []toolcall.ParsedToolCall
ToolCallDeltas []toolCallDelta ToolCallDeltas []toolCallDelta
} }

View File

@@ -1,10 +1,10 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"regexp" "regexp"
"strings" "strings"
"ds2api/internal/util"
) )
// --- XML tool call support for the streaming sieve --- // --- XML tool call support for the streaming sieve ---
@@ -43,7 +43,7 @@ var xmlToolTagsToDetect = []string{"<tool_calls>", "<tool_calls\n", "<tool_call>
"<attempt_completion>", "<ask_followup_question>", "<new_task>"} "<attempt_completion>", "<ask_followup_question>", "<new_task>"}
// consumeXMLToolCapture tries to extract complete XML tool call blocks from captured text. // consumeXMLToolCapture tries to extract complete XML tool call blocks from captured text.
func consumeXMLToolCapture(captured string, toolNames []string) (prefix string, calls []util.ParsedToolCall, suffix string, ready bool) { func consumeXMLToolCapture(captured string, toolNames []string) (prefix string, calls []toolcall.ParsedToolCall, suffix string, ready bool) {
lower := strings.ToLower(captured) lower := strings.ToLower(captured)
// Find the FIRST matching open/close pair, preferring wrapper tags. // Find the FIRST matching open/close pair, preferring wrapper tags.
// Tag pairs are ordered longest-first (e.g. <tool_calls before <tool_call) // Tag pairs are ordered longest-first (e.g. <tool_calls before <tool_call)
@@ -66,7 +66,7 @@ func consumeXMLToolCapture(captured string, toolNames []string) (prefix string,
xmlBlock := captured[openIdx:closeEnd] xmlBlock := captured[openIdx:closeEnd]
prefixPart := captured[:openIdx] prefixPart := captured[:openIdx]
suffixPart := captured[closeEnd:] suffixPart := captured[closeEnd:]
parsed := util.ParseToolCalls(xmlBlock, toolNames) parsed := toolcall.ParseToolCalls(xmlBlock, toolNames)
if len(parsed) > 0 { if len(parsed) > 0 {
prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart) prefixPart, suffixPart = trimWrappingJSONFence(prefixPart, suffixPart)
return prefixPart, parsed, suffixPart, true return prefixPart, parsed, suffixPart, true

View File

@@ -1,6 +1,7 @@
package compat package compat
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"os" "os"
"path/filepath" "path/filepath"
@@ -86,22 +87,22 @@ func TestGoCompatToolcallFixtures(t *testing.T) {
mustLoadJSON(t, fixturePath, &fixture) mustLoadJSON(t, fixturePath, &fixture)
var expected struct { var expected struct {
Calls []util.ParsedToolCall `json:"calls"` Calls []toolcall.ParsedToolCall `json:"calls"`
SawToolCallSyntax bool `json:"sawToolCallSyntax"` SawToolCallSyntax bool `json:"sawToolCallSyntax"`
RejectedByPolicy bool `json:"rejectedByPolicy"` RejectedByPolicy bool `json:"rejectedByPolicy"`
RejectedToolNames []string `json:"rejectedToolNames"` RejectedToolNames []string `json:"rejectedToolNames"`
} }
mustLoadJSON(t, expectedPath, &expected) mustLoadJSON(t, expectedPath, &expected)
var got util.ToolCallParseResult var got toolcall.ToolCallParseResult
switch strings.ToLower(strings.TrimSpace(fixture.Mode)) { switch strings.ToLower(strings.TrimSpace(fixture.Mode)) {
case "standalone": case "standalone":
got = util.ParseStandaloneToolCallsDetailed(fixture.Text, fixture.ToolNames) got = toolcall.ParseStandaloneToolCallsDetailed(fixture.Text, fixture.ToolNames)
default: default:
got = util.ParseToolCallsDetailed(fixture.Text, fixture.ToolNames) got = toolcall.ParseToolCallsDetailed(fixture.Text, fixture.ToolNames)
} }
if got.Calls == nil { if got.Calls == nil {
got.Calls = []util.ParsedToolCall{} got.Calls = []toolcall.ParsedToolCall{}
} }
if got.RejectedToolNames == nil { if got.RejectedToolNames == nil {
got.RejectedToolNames = []string{} got.RejectedToolNames = []string{}

View File

@@ -1,6 +1,7 @@
package claude package claude
import ( import (
"ds2api/internal/toolcall"
"fmt" "fmt"
"time" "time"
@@ -8,9 +9,9 @@ import (
) )
func BuildMessageResponse(messageID, model string, normalizedMessages []any, finalThinking, finalText string, toolNames []string) map[string]any { func BuildMessageResponse(messageID, model string, normalizedMessages []any, finalThinking, finalText string, toolNames []string) map[string]any {
detected := util.ParseToolCalls(finalText, toolNames) detected := toolcall.ParseToolCalls(finalText, toolNames)
if len(detected) == 0 && finalText == "" && finalThinking != "" { if len(detected) == 0 && finalText == "" && finalThinking != "" {
detected = util.ParseToolCalls(finalThinking, toolNames) detected = toolcall.ParseToolCalls(finalThinking, toolNames)
} }
content := make([]map[string]any, 0, 4) content := make([]map[string]any, 0, 4)
if finalThinking != "" { if finalThinking != "" {

View File

@@ -1,14 +1,14 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"strings" "strings"
"time" "time"
"ds2api/internal/util"
) )
func BuildChatCompletion(completionID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any { func BuildChatCompletion(completionID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any {
detected := util.ParseStandaloneToolCallsDetailed(finalText, toolNames) detected := toolcall.ParseStandaloneToolCallsDetailed(finalText, toolNames)
finishReason := "stop" finishReason := "stop"
messageObj := map[string]any{"role": "assistant", "content": finalText} messageObj := map[string]any{"role": "assistant", "content": finalText}
if strings.TrimSpace(finalThinking) != "" { if strings.TrimSpace(finalThinking) != "" {
@@ -16,7 +16,7 @@ func BuildChatCompletion(completionID, model, finalPrompt, finalThinking, finalT
} }
if len(detected.Calls) > 0 { if len(detected.Calls) > 0 {
finishReason = "tool_calls" finishReason = "tool_calls"
messageObj["tool_calls"] = util.FormatOpenAIToolCalls(detected.Calls) messageObj["tool_calls"] = toolcall.FormatOpenAIToolCalls(detected.Calls)
messageObj["content"] = nil messageObj["content"] = nil
} }

View File

@@ -1,19 +1,19 @@
package openai package openai
import ( import (
"ds2api/internal/toolcall"
"encoding/json" "encoding/json"
"strings" "strings"
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
"ds2api/internal/util"
) )
func BuildResponseObject(responseID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any { func BuildResponseObject(responseID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any {
// Strict mode: only standalone, structured tool-call payloads are treated // Strict mode: only standalone, structured tool-call payloads are treated
// as executable tool calls. // as executable tool calls.
detected := util.ParseStandaloneToolCallsDetailed(finalText, toolNames) detected := toolcall.ParseStandaloneToolCallsDetailed(finalText, toolNames)
exposedOutputText := finalText exposedOutputText := finalText
output := make([]any, 0, 2) output := make([]any, 0, 2)
if len(detected.Calls) > 0 { if len(detected.Calls) > 0 {
@@ -71,7 +71,7 @@ func BuildResponseObjectFromItems(responseID, model, finalPrompt, finalThinking,
} }
} }
func toResponsesFunctionCallItems(toolCalls []util.ParsedToolCall) []any { func toResponsesFunctionCallItems(toolCalls []toolcall.ParsedToolCall) []any {
if len(toolCalls) == 0 { if len(toolCalls) == 0 {
return nil return nil
} }

View File

@@ -1,4 +1,4 @@
package util package toolcall
import "strings" import "strings"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"strings" "strings"

View File

@@ -0,0 +1,92 @@
package toolcall
import (
"testing"
)
// ─── FormatOpenAIStreamToolCalls ─────────────────────────────────────
func TestFormatOpenAIStreamToolCalls(t *testing.T) {
formatted := FormatOpenAIStreamToolCalls([]ParsedToolCall{
{Name: "search", Input: map[string]any{"q": "test"}},
})
if len(formatted) != 1 {
t.Fatalf("expected 1, got %d", len(formatted))
}
fn, _ := formatted[0]["function"].(map[string]any)
if fn["name"] != "search" {
t.Fatalf("unexpected function name: %#v", fn)
}
if formatted[0]["index"] != 0 {
t.Fatalf("expected index 0, got %v", formatted[0]["index"])
}
}
// ─── ParseToolCalls more edge cases ──────────────────────────────────
func TestParseToolCallsNoToolNames(t *testing.T) {
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}`
calls := ParseToolCalls(text, nil)
if len(calls) != 1 {
t.Fatalf("expected 1 call with nil tool names, got %d", len(calls))
}
}
func TestParseToolCallsEmptyText(t *testing.T) {
calls := ParseToolCalls("", []string{"search"})
if len(calls) != 0 {
t.Fatalf("expected 0 calls for empty text, got %d", len(calls))
}
}
func TestParseToolCallsMultipleTools(t *testing.T) {
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}},{"name":"get_weather","input":{"city":"beijing"}}]}`
calls := ParseToolCalls(text, []string{"search", "get_weather"})
if len(calls) != 2 {
t.Fatalf("expected 2 calls, got %d", len(calls))
}
}
func TestParseToolCallsInputAsString(t *testing.T) {
text := `{"tool_calls":[{"name":"search","input":"{\"q\":\"golang\"}"}]}`
calls := ParseToolCalls(text, []string{"search"})
if len(calls) != 1 {
t.Fatalf("expected 1 call, got %d", len(calls))
}
if calls[0].Input["q"] != "golang" {
t.Fatalf("expected parsed string input, got %#v", calls[0].Input)
}
}
func TestParseToolCallsWithFunctionWrapper(t *testing.T) {
text := `{"tool_calls":[{"function":{"name":"calc","arguments":{"x":1,"y":2}}}]}`
calls := ParseToolCalls(text, []string{"calc"})
if len(calls) != 1 {
t.Fatalf("expected 1 call, got %d", len(calls))
}
if calls[0].Name != "calc" {
t.Fatalf("expected calc, got %q", calls[0].Name)
}
}
func TestParseStandaloneToolCallsFencedCodeBlock(t *testing.T) {
fenced := "Here's an example:\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```\nDon't execute this."
calls := ParseStandaloneToolCalls(fenced, []string{"search"})
if len(calls) != 0 {
t.Fatalf("expected fenced code block to be ignored, got %d calls", len(calls))
}
}
// ─── looksLikeToolExampleContext ─────────────────────────────────────
func TestLooksLikeToolExampleContextNone(t *testing.T) {
if looksLikeToolExampleContext("I will call the tool now") {
t.Fatal("expected false for non-example context")
}
}
func TestLooksLikeToolExampleContextFenced(t *testing.T) {
if !looksLikeToolExampleContext("```json") {
t.Fatal("expected true for fenced code block context")
}
}

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"regexp" "regexp"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"encoding/json" "encoding/json"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"encoding/json" "encoding/json"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"regexp" "regexp"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"encoding/json" "encoding/json"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"regexp" "regexp"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"encoding/json" "encoding/json"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import "strings" import "strings"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"encoding/json" "encoding/json"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"strings" "strings"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"regexp" "regexp"

View File

@@ -1,4 +1,4 @@
package util package toolcall
import ( import (
"testing" "testing"

View File

@@ -1,6 +1,7 @@
package util package util
import ( import (
"ds2api/internal/toolcall"
"fmt" "fmt"
"strings" "strings"
"time" "time"
@@ -11,7 +12,7 @@ import (
// BuildOpenAIChatCompletion is kept for backward compatibility. // BuildOpenAIChatCompletion is kept for backward compatibility.
// Prefer internal/format/openai.BuildChatCompletion for new code. // Prefer internal/format/openai.BuildChatCompletion for new code.
func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any { func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any {
detected := ParseToolCalls(finalText, toolNames) detected := toolcall.ParseToolCalls(finalText, toolNames)
finishReason := "stop" finishReason := "stop"
messageObj := map[string]any{"role": "assistant", "content": finalText} messageObj := map[string]any{"role": "assistant", "content": finalText}
if strings.TrimSpace(finalThinking) != "" { if strings.TrimSpace(finalThinking) != "" {
@@ -19,7 +20,7 @@ func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking,
} }
if len(detected) > 0 { if len(detected) > 0 {
finishReason = "tool_calls" finishReason = "tool_calls"
messageObj["tool_calls"] = FormatOpenAIToolCalls(detected) messageObj["tool_calls"] = toolcall.FormatOpenAIToolCalls(detected)
messageObj["content"] = nil messageObj["content"] = nil
} }
promptTokens := EstimateTokens(finalPrompt) promptTokens := EstimateTokens(finalPrompt)
@@ -46,7 +47,7 @@ func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking,
// BuildOpenAIResponseObject is kept for backward compatibility. // BuildOpenAIResponseObject is kept for backward compatibility.
// Prefer internal/format/openai.BuildResponseObject for new code. // Prefer internal/format/openai.BuildResponseObject for new code.
func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any { func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any {
detected := ParseToolCalls(finalText, toolNames) detected := toolcall.ParseToolCalls(finalText, toolNames)
exposedOutputText := finalText exposedOutputText := finalText
output := make([]any, 0, 2) output := make([]any, 0, 2)
if len(detected) > 0 { if len(detected) > 0 {
@@ -56,9 +57,9 @@ func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, fi
toolCalls := make([]any, 0, len(detected)) toolCalls := make([]any, 0, len(detected))
for _, tc := range detected { for _, tc := range detected {
toolCalls = append(toolCalls, map[string]any{ toolCalls = append(toolCalls, map[string]any{
"type": "tool_call", "type": "tool_call",
"name": tc.Name, "name": tc.Name,
"arguments": tc.Input, "arguments": tc.Input,
}) })
} }
output = append(output, map[string]any{ output = append(output, map[string]any{
@@ -108,7 +109,7 @@ func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, fi
// BuildClaudeMessageResponse is kept for backward compatibility. // BuildClaudeMessageResponse is kept for backward compatibility.
// Prefer internal/format/claude.BuildMessageResponse for new code. // Prefer internal/format/claude.BuildMessageResponse for new code.
func BuildClaudeMessageResponse(messageID, model string, normalizedMessages []any, finalThinking, finalText string, toolNames []string) map[string]any { func BuildClaudeMessageResponse(messageID, model string, normalizedMessages []any, finalThinking, finalText string, toolNames []string) map[string]any {
detected := ParseToolCalls(finalText, toolNames) detected := toolcall.ParseToolCalls(finalText, toolNames)
content := make([]map[string]any, 0, 4) content := make([]map[string]any, 0, 4)
if finalThinking != "" { if finalThinking != "" {
content = append(content, map[string]any{"type": "thinking", "thinking": finalThinking}) content = append(content, map[string]any{"type": "thinking", "thinking": finalThinking})

View File

@@ -356,89 +356,3 @@ func TestConvertClaudeToDeepSeekOpusUsesSlowMapping(t *testing.T) {
} }
} }
// ─── FormatOpenAIStreamToolCalls ─────────────────────────────────────
func TestFormatOpenAIStreamToolCalls(t *testing.T) {
formatted := FormatOpenAIStreamToolCalls([]ParsedToolCall{
{Name: "search", Input: map[string]any{"q": "test"}},
})
if len(formatted) != 1 {
t.Fatalf("expected 1, got %d", len(formatted))
}
fn, _ := formatted[0]["function"].(map[string]any)
if fn["name"] != "search" {
t.Fatalf("unexpected function name: %#v", fn)
}
if formatted[0]["index"] != 0 {
t.Fatalf("expected index 0, got %v", formatted[0]["index"])
}
}
// ─── ParseToolCalls more edge cases ──────────────────────────────────
func TestParseToolCallsNoToolNames(t *testing.T) {
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}`
calls := ParseToolCalls(text, nil)
if len(calls) != 1 {
t.Fatalf("expected 1 call with nil tool names, got %d", len(calls))
}
}
func TestParseToolCallsEmptyText(t *testing.T) {
calls := ParseToolCalls("", []string{"search"})
if len(calls) != 0 {
t.Fatalf("expected 0 calls for empty text, got %d", len(calls))
}
}
func TestParseToolCallsMultipleTools(t *testing.T) {
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}},{"name":"get_weather","input":{"city":"beijing"}}]}`
calls := ParseToolCalls(text, []string{"search", "get_weather"})
if len(calls) != 2 {
t.Fatalf("expected 2 calls, got %d", len(calls))
}
}
func TestParseToolCallsInputAsString(t *testing.T) {
text := `{"tool_calls":[{"name":"search","input":"{\"q\":\"golang\"}"}]}`
calls := ParseToolCalls(text, []string{"search"})
if len(calls) != 1 {
t.Fatalf("expected 1 call, got %d", len(calls))
}
if calls[0].Input["q"] != "golang" {
t.Fatalf("expected parsed string input, got %#v", calls[0].Input)
}
}
func TestParseToolCallsWithFunctionWrapper(t *testing.T) {
text := `{"tool_calls":[{"function":{"name":"calc","arguments":{"x":1,"y":2}}}]}`
calls := ParseToolCalls(text, []string{"calc"})
if len(calls) != 1 {
t.Fatalf("expected 1 call, got %d", len(calls))
}
if calls[0].Name != "calc" {
t.Fatalf("expected calc, got %q", calls[0].Name)
}
}
func TestParseStandaloneToolCallsFencedCodeBlock(t *testing.T) {
fenced := "Here's an example:\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```\nDon't execute this."
calls := ParseStandaloneToolCalls(fenced, []string{"search"})
if len(calls) != 0 {
t.Fatalf("expected fenced code block to be ignored, got %d calls", len(calls))
}
}
// ─── looksLikeToolExampleContext ─────────────────────────────────────
func TestLooksLikeToolExampleContextNone(t *testing.T) {
if looksLikeToolExampleContext("I will call the tool now") {
t.Fatal("expected false for non-example context")
}
}
func TestLooksLikeToolExampleContextFenced(t *testing.T) {
if !looksLikeToolExampleContext("```json") {
t.Fatal("expected true for fenced code block context")
}
}