mirror of
https://github.com/CJackHwang/ds2api.git
synced 2026-05-08 02:15:27 +08:00
refactor: move tool call parsing and formatting logic to a dedicated internal/toolcall package
This commit is contained in:
CJACK
@@ -1,6 +1,7 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"ds2api/internal/toolcall"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
// BuildOpenAIChatCompletion is kept for backward compatibility.
|
||||
// Prefer internal/format/openai.BuildChatCompletion for new code.
|
||||
func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any {
|
||||
detected := ParseToolCalls(finalText, toolNames)
|
||||
detected := toolcall.ParseToolCalls(finalText, toolNames)
|
||||
finishReason := "stop"
|
||||
messageObj := map[string]any{"role": "assistant", "content": finalText}
|
||||
if strings.TrimSpace(finalThinking) != "" {
|
||||
@@ -19,7 +20,7 @@ func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking,
|
||||
}
|
||||
if len(detected) > 0 {
|
||||
finishReason = "tool_calls"
|
||||
messageObj["tool_calls"] = FormatOpenAIToolCalls(detected)
|
||||
messageObj["tool_calls"] = toolcall.FormatOpenAIToolCalls(detected)
|
||||
messageObj["content"] = nil
|
||||
}
|
||||
promptTokens := EstimateTokens(finalPrompt)
|
||||
@@ -46,7 +47,7 @@ func BuildOpenAIChatCompletion(completionID, model, finalPrompt, finalThinking,
|
||||
// BuildOpenAIResponseObject is kept for backward compatibility.
|
||||
// Prefer internal/format/openai.BuildResponseObject for new code.
|
||||
func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, finalText string, toolNames []string) map[string]any {
|
||||
detected := ParseToolCalls(finalText, toolNames)
|
||||
detected := toolcall.ParseToolCalls(finalText, toolNames)
|
||||
exposedOutputText := finalText
|
||||
output := make([]any, 0, 2)
|
||||
if len(detected) > 0 {
|
||||
@@ -56,9 +57,9 @@ func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, fi
|
||||
toolCalls := make([]any, 0, len(detected))
|
||||
for _, tc := range detected {
|
||||
toolCalls = append(toolCalls, map[string]any{
|
||||
"type": "tool_call",
|
||||
"name": tc.Name,
|
||||
"arguments": tc.Input,
|
||||
"type": "tool_call",
|
||||
"name": tc.Name,
|
||||
"arguments": tc.Input,
|
||||
})
|
||||
}
|
||||
output = append(output, map[string]any{
|
||||
@@ -108,7 +109,7 @@ func BuildOpenAIResponseObject(responseID, model, finalPrompt, finalThinking, fi
|
||||
// BuildClaudeMessageResponse is kept for backward compatibility.
|
||||
// Prefer internal/format/claude.BuildMessageResponse for new code.
|
||||
func BuildClaudeMessageResponse(messageID, model string, normalizedMessages []any, finalThinking, finalText string, toolNames []string) map[string]any {
|
||||
detected := ParseToolCalls(finalText, toolNames)
|
||||
detected := toolcall.ParseToolCalls(finalText, toolNames)
|
||||
content := make([]map[string]any, 0, 4)
|
||||
if finalThinking != "" {
|
||||
content = append(content, map[string]any{"type": "thinking", "thinking": finalThinking})
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
package util
|
||||
|
||||
import "strings"
|
||||
|
||||
// BuildToolCallInstructions generates the unified tool-calling instruction block
|
||||
// used by all adapters (OpenAI, Claude, Gemini). It uses attention-optimized
|
||||
// structure: rules → negative examples → positive examples → anchor.
|
||||
//
|
||||
// The toolNames slice should contain the actual tool names available in the
|
||||
// current request; the function picks real names for examples.
|
||||
func BuildToolCallInstructions(toolNames []string) string {
|
||||
// Pick real tool names for examples; fall back to generic names.
|
||||
ex1 := "read_file"
|
||||
ex2 := "write_to_file"
|
||||
ex3 := "ask_followup_question"
|
||||
used := map[string]bool{}
|
||||
for _, n := range toolNames {
|
||||
switch {
|
||||
// Read/query-type tools
|
||||
case !used["ex1"] && matchAny(n, "read_file", "list_files", "search_files", "Read", "Glob"):
|
||||
ex1 = n
|
||||
used["ex1"] = true
|
||||
// Write/execute-type tools
|
||||
case !used["ex2"] && matchAny(n, "write_to_file", "apply_diff", "execute_command", "exec_command", "Write", "Edit", "MultiEdit", "Bash"):
|
||||
ex2 = n
|
||||
used["ex2"] = true
|
||||
// Interactive/meta tools
|
||||
case !used["ex3"] && matchAny(n, "ask_followup_question", "attempt_completion", "update_todo_list", "Task"):
|
||||
ex3 = n
|
||||
used["ex3"] = true
|
||||
}
|
||||
}
|
||||
ex1Params := exampleReadParams(ex1)
|
||||
ex2Params := exampleWriteOrExecParams(ex2)
|
||||
ex3Params := exampleInteractiveParams(ex3)
|
||||
|
||||
return `TOOL CALL FORMAT — FOLLOW EXACTLY:
|
||||
|
||||
When calling tools, emit ONLY raw XML at the very end of your response. No text before, no text after, no markdown fences.
|
||||
|
||||
<tool_calls>
|
||||
<tool_call>
|
||||
<tool_name>TOOL_NAME_HERE</tool_name>
|
||||
<parameters>{"key":"value"}</parameters>
|
||||
</tool_call>
|
||||
</tool_calls>
|
||||
|
||||
RULES:
|
||||
1) Output ONLY the XML above when calling tools. Do NOT mix tool XML with regular text.
|
||||
2) <parameters> MUST contain a strict JSON object. All JSON keys and strings use double quotes.
|
||||
3) Multiple tools → multiple <tool_call> blocks inside ONE <tool_calls> root.
|
||||
4) Do NOT wrap the XML in markdown code fences (no triple backticks).
|
||||
5) After receiving a tool result, use it directly. Only call another tool if the result is insufficient.
|
||||
6) Parameters MUST use the exact field names from the selected tool schema.
|
||||
7) CRITICAL: Do NOT invent or add any extra fields (such as "_raw", "_xml"). Use ONLY the fields strictly defined in the schema. Extra fields will cause execution failure.
|
||||
|
||||
❌ WRONG — Do NOT do these:
|
||||
Wrong 1 — mixed text and XML:
|
||||
I'll read the file for you. <tool_calls><tool_call>...
|
||||
Wrong 2 — describing tool calls in text:
|
||||
[调用 Bash] {"command": "ls"}
|
||||
Wrong 3 — missing <tool_calls> wrapper:
|
||||
<tool_call><tool_name>` + ex1 + `</tool_name><parameters>{}</parameters></tool_call>
|
||||
Wrong 4 — extra/invented fields:
|
||||
<parameters>{"_raw": "...", "command": "ls"}</parameters>
|
||||
|
||||
|
||||
✅ CORRECT EXAMPLES:
|
||||
|
||||
Example A — Single tool:
|
||||
<tool_calls>
|
||||
<tool_call>
|
||||
<tool_name>` + ex1 + `</tool_name>
|
||||
<parameters>` + ex1Params + `</parameters>
|
||||
</tool_call>
|
||||
</tool_calls>
|
||||
|
||||
Example B — Two tools in parallel:
|
||||
<tool_calls>
|
||||
<tool_call>
|
||||
<tool_name>` + ex1 + `</tool_name>
|
||||
<parameters>` + ex1Params + `</parameters>
|
||||
</tool_call>
|
||||
<tool_call>
|
||||
<tool_name>` + ex2 + `</tool_name>
|
||||
<parameters>` + ex2Params + `</parameters>
|
||||
</tool_call>
|
||||
</tool_calls>
|
||||
|
||||
Example C — Tool with complex nested JSON parameters:
|
||||
<tool_calls>
|
||||
<tool_call>
|
||||
<tool_name>` + ex3 + `</tool_name>
|
||||
<parameters>` + ex3Params + `</parameters>
|
||||
</tool_call>
|
||||
</tool_calls>
|
||||
|
||||
Remember: Output ONLY the <tool_calls>...</tool_calls> XML block when calling tools.`
|
||||
}
|
||||
|
||||
func matchAny(name string, candidates ...string) bool {
|
||||
for _, c := range candidates {
|
||||
if name == c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func exampleReadParams(name string) string {
|
||||
switch strings.TrimSpace(name) {
|
||||
case "Read":
|
||||
return `{"file_path":"README.md"}`
|
||||
case "Glob":
|
||||
return `{"pattern":"**/*.go","path":"."}`
|
||||
default:
|
||||
return `{"path":"src/main.go"}`
|
||||
}
|
||||
}
|
||||
|
||||
func exampleWriteOrExecParams(name string) string {
|
||||
switch strings.TrimSpace(name) {
|
||||
case "Bash", "execute_command":
|
||||
return `{"command":"pwd"}`
|
||||
case "exec_command":
|
||||
return `{"cmd":"pwd"}`
|
||||
case "Edit":
|
||||
return `{"file_path":"README.md","old_string":"foo","new_string":"bar"}`
|
||||
case "MultiEdit":
|
||||
return `{"file_path":"README.md","edits":[{"old_string":"foo","new_string":"bar"}]}`
|
||||
default:
|
||||
return `{"path":"output.txt","content":"Hello world"}`
|
||||
}
|
||||
}
|
||||
|
||||
func exampleInteractiveParams(name string) string {
|
||||
switch strings.TrimSpace(name) {
|
||||
case "Task":
|
||||
return `{"description":"Investigate flaky tests","prompt":"Run targeted tests and summarize failures"}`
|
||||
default:
|
||||
return `{"question":"Which approach do you prefer?","follow_up":[{"text":"Option A"},{"text":"Option B"}]}`
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildToolCallInstructions_ExecCommandUsesCmdExample(t *testing.T) {
|
||||
out := BuildToolCallInstructions([]string{"exec_command"})
|
||||
if !strings.Contains(out, `<tool_name>exec_command</tool_name>`) {
|
||||
t.Fatalf("expected exec_command in examples, got: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `<parameters>{"cmd":"pwd"}</parameters>`) {
|
||||
t.Fatalf("expected cmd parameter example for exec_command, got: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildToolCallInstructions_ExecuteCommandUsesCommandExample(t *testing.T) {
|
||||
out := BuildToolCallInstructions([]string{"execute_command"})
|
||||
if !strings.Contains(out, `<tool_name>execute_command</tool_name>`) {
|
||||
t.Fatalf("expected execute_command in examples, got: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `<parameters>{"command":"pwd"}</parameters>`) {
|
||||
t.Fatalf("expected command parameter example for execute_command, got: %s", out)
|
||||
}
|
||||
}
|
||||
@@ -1,202 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var toolCallPattern = regexp.MustCompile(`\{\s*["']tool_calls["']\s*:\s*\[(.*?)\]\s*\}`)
|
||||
var fencedJSONPattern = regexp.MustCompile("(?s)```(?:json)?\\s*(.*?)\\s*```")
|
||||
var fencedCodeBlockPattern = regexp.MustCompile("(?s)```[\\s\\S]*?```")
|
||||
var markupToolSyntaxPattern = regexp.MustCompile(`(?i)<(?:(?:[a-z0-9_:-]+:)?(?:tool_call|function_call|invoke)\b|(?:[a-z0-9_:-]+:)?function_calls\b|(?:[a-z0-9_:-]+:)?tool_use\b)`)
|
||||
|
||||
func buildToolCallCandidates(text string) []string {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
candidates := []string{trimmed}
|
||||
|
||||
// fenced code block candidates: ```json ... ```
|
||||
for _, match := range fencedJSONPattern.FindAllStringSubmatch(trimmed, -1) {
|
||||
if len(match) >= 2 {
|
||||
candidates = append(candidates, strings.TrimSpace(match[1]))
|
||||
}
|
||||
}
|
||||
|
||||
// best-effort extraction around tool call keywords in mixed text payloads.
|
||||
candidates = append(candidates, extractToolCallObjects(trimmed)...)
|
||||
|
||||
// best-effort object slice: from first '{' to last '}'
|
||||
first := strings.Index(trimmed, "{")
|
||||
last := strings.LastIndex(trimmed, "}")
|
||||
if first >= 0 && last > first {
|
||||
candidates = append(candidates, strings.TrimSpace(trimmed[first:last+1]))
|
||||
}
|
||||
// best-effort array slice: from first '[' to last ']'
|
||||
firstArr := strings.Index(trimmed, "[")
|
||||
lastArr := strings.LastIndex(trimmed, "]")
|
||||
if firstArr >= 0 && lastArr > firstArr {
|
||||
candidates = append(candidates, strings.TrimSpace(trimmed[firstArr:lastArr+1]))
|
||||
}
|
||||
|
||||
// legacy regex extraction fallback
|
||||
if m := toolCallPattern.FindStringSubmatch(trimmed); len(m) >= 2 {
|
||||
candidates = append(candidates, "{"+`"tool_calls":[`+m[1]+"]}")
|
||||
}
|
||||
|
||||
uniq := make([]string, 0, len(candidates))
|
||||
seen := map[string]struct{}{}
|
||||
for _, c := range candidates {
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[c]; ok {
|
||||
continue
|
||||
}
|
||||
seen[c] = struct{}{}
|
||||
uniq = append(uniq, c)
|
||||
}
|
||||
return uniq
|
||||
}
|
||||
|
||||
func extractToolCallObjects(text string) []string {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
out := []string{}
|
||||
offset := 0
|
||||
keywords := []string{"tool_calls", "\"function\"", "function.name:", "functioncall", "\"tool_use\""}
|
||||
for {
|
||||
bestIdx := -1
|
||||
matchedKeyword := ""
|
||||
for _, kw := range keywords {
|
||||
idx := strings.Index(lower[offset:], kw)
|
||||
if idx >= 0 {
|
||||
absIdx := offset + idx
|
||||
if bestIdx < 0 || absIdx < bestIdx {
|
||||
bestIdx = absIdx
|
||||
matchedKeyword = kw
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if bestIdx < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
idx := bestIdx
|
||||
// Avoid backtracking too far to prevent OOM on malicious or very long strings
|
||||
searchLimit := idx - 2000
|
||||
if searchLimit < offset {
|
||||
searchLimit = offset
|
||||
}
|
||||
|
||||
start := strings.LastIndex(text[searchLimit:idx], "{")
|
||||
if start >= 0 {
|
||||
start += searchLimit
|
||||
}
|
||||
|
||||
if start < 0 {
|
||||
offset = idx + len(matchedKeyword)
|
||||
continue
|
||||
}
|
||||
|
||||
foundObj := false
|
||||
for start >= searchLimit {
|
||||
candidate, end, ok := extractJSONObject(text, start)
|
||||
if ok {
|
||||
// Move forward to avoid repeatedly matching the same object.
|
||||
offset = end
|
||||
out = append(out, strings.TrimSpace(candidate))
|
||||
foundObj = true
|
||||
break
|
||||
}
|
||||
// Try previous '{'
|
||||
if start > searchLimit {
|
||||
prevStart := strings.LastIndex(text[searchLimit:start], "{")
|
||||
if prevStart >= 0 {
|
||||
start = searchLimit + prevStart
|
||||
continue
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if !foundObj {
|
||||
offset = idx + len(matchedKeyword)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractJSONObject(text string, start int) (string, int, bool) {
|
||||
if start < 0 || start >= len(text) || text[start] != '{' {
|
||||
return "", 0, false
|
||||
}
|
||||
depth := 0
|
||||
quote := byte(0)
|
||||
escaped := false
|
||||
// Limit scan length to avoid OOM on unclosed objects
|
||||
maxLen := start + 50000
|
||||
if maxLen > len(text) {
|
||||
maxLen = len(text)
|
||||
}
|
||||
for i := start; i < maxLen; i++ {
|
||||
ch := text[i]
|
||||
if quote != 0 {
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if ch == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if ch == quote {
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ch == '"' || ch == '\'' {
|
||||
quote = ch
|
||||
continue
|
||||
}
|
||||
if ch == '{' {
|
||||
depth++
|
||||
continue
|
||||
}
|
||||
if ch == '}' {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return text[start : i+1], i + 1, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", 0, false
|
||||
}
|
||||
|
||||
func looksLikeToolExampleContext(text string) bool {
|
||||
t := strings.ToLower(strings.TrimSpace(text))
|
||||
if t == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(t, "```")
|
||||
}
|
||||
|
||||
func shouldSkipToolCallParsingForCodeFenceExample(text string) bool {
|
||||
if !looksLikeToolCallSyntax(text) {
|
||||
return false
|
||||
}
|
||||
stripped := strings.TrimSpace(stripFencedCodeBlocks(text))
|
||||
return !looksLikeToolCallSyntax(stripped)
|
||||
}
|
||||
|
||||
func looksLikeMarkupToolSyntax(text string) bool {
|
||||
return markupToolSyntaxPattern.MatchString(text)
|
||||
}
|
||||
|
||||
func stripFencedCodeBlocks(text string) string {
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
return fencedCodeBlockPattern.ReplaceAllString(text, " ")
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func FormatOpenAIToolCalls(calls []ParsedToolCall) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(calls))
|
||||
for _, c := range calls {
|
||||
args, _ := json.Marshal(c.Input)
|
||||
out = append(out, map[string]any{
|
||||
"id": "call_" + strings.ReplaceAll(uuid.NewString(), "-", ""),
|
||||
"type": "function",
|
||||
"function": map[string]any{
|
||||
"name": c.Name,
|
||||
"arguments": string(args),
|
||||
},
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func FormatOpenAIStreamToolCalls(calls []ParsedToolCall) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(calls))
|
||||
for i, c := range calls {
|
||||
args, _ := json.Marshal(c.Input)
|
||||
out = append(out, map[string]any{
|
||||
"index": i,
|
||||
"id": "call_" + strings.ReplaceAll(uuid.NewString(), "-", ""),
|
||||
"type": "function",
|
||||
"function": map[string]any{
|
||||
"name": c.Name,
|
||||
"arguments": string(args),
|
||||
},
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
func parseToolCallInput(v any) map[string]any {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return map[string]any{}
|
||||
case map[string]any:
|
||||
return x
|
||||
case string:
|
||||
raw := strings.TrimSpace(x)
|
||||
if raw == "" {
|
||||
return map[string]any{}
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &parsed); err == nil && parsed != nil {
|
||||
repairPathLikeControlChars(parsed)
|
||||
return parsed
|
||||
}
|
||||
// Try to repair invalid backslashes (common in Windows paths output by models)
|
||||
repaired := repairInvalidJSONBackslashes(raw)
|
||||
if repaired != raw {
|
||||
if err := json.Unmarshal([]byte(repaired), &parsed); err == nil && parsed != nil {
|
||||
repairPathLikeControlChars(parsed)
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
// Try to repair loose JSON in string argument as well
|
||||
repairedLoose := RepairLooseJSON(raw)
|
||||
if repairedLoose != raw {
|
||||
if err := json.Unmarshal([]byte(repairedLoose), &parsed); err == nil && parsed != nil {
|
||||
repairPathLikeControlChars(parsed)
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return map[string]any{"_raw": raw}
|
||||
default:
|
||||
b, err := json.Marshal(x)
|
||||
if err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal(b, &parsed); err == nil && parsed != nil {
|
||||
return parsed
|
||||
}
|
||||
return map[string]any{}
|
||||
}
|
||||
}
|
||||
|
||||
func repairPathLikeControlChars(m map[string]any) {
|
||||
for k, v := range m {
|
||||
switch vv := v.(type) {
|
||||
case map[string]any:
|
||||
repairPathLikeControlChars(vv)
|
||||
case []any:
|
||||
for _, item := range vv {
|
||||
if child, ok := item.(map[string]any); ok {
|
||||
repairPathLikeControlChars(child)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if isPathLikeKey(k) && containsControlRune(vv) {
|
||||
m[k] = escapeControlRunes(vv)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isPathLikeKey(key string) bool {
|
||||
k := strings.ToLower(strings.TrimSpace(key))
|
||||
return strings.Contains(k, "path") || strings.Contains(k, "file")
|
||||
}
|
||||
|
||||
func containsControlRune(s string) bool {
|
||||
for _, r := range s {
|
||||
if unicode.IsControl(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func escapeControlRunes(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s) + 8)
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '\b':
|
||||
b.WriteString(`\b`)
|
||||
case '\f':
|
||||
b.WriteString(`\f`)
|
||||
case '\n':
|
||||
b.WriteString(`\n`)
|
||||
case '\r':
|
||||
b.WriteString(`\r`)
|
||||
case '\t':
|
||||
b.WriteString(`\t`)
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func repairInvalidJSONBackslashes(s string) string {
|
||||
if !strings.Contains(s, "\\") {
|
||||
return s
|
||||
}
|
||||
var out strings.Builder
|
||||
out.Grow(len(s) + 10)
|
||||
runes := []rune(s)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
if runes[i] == '\\' {
|
||||
if i+1 < len(runes) {
|
||||
next := runes[i+1]
|
||||
switch next {
|
||||
case '"', '\\', '/', 'b', 'f', 'n', 'r', 't':
|
||||
out.WriteRune('\\')
|
||||
out.WriteRune(next)
|
||||
i++
|
||||
continue
|
||||
case 'u':
|
||||
if i+5 < len(runes) {
|
||||
isHex := true
|
||||
for j := 1; j <= 4; j++ {
|
||||
r := runes[i+1+j]
|
||||
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
|
||||
isHex = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if isHex {
|
||||
out.WriteRune('\\')
|
||||
out.WriteRune('u')
|
||||
for j := 1; j <= 4; j++ {
|
||||
out.WriteRune(runes[i+1+j])
|
||||
}
|
||||
i += 5
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Not a valid escape sequence, double it
|
||||
out.WriteString("\\\\")
|
||||
} else {
|
||||
out.WriteRune(runes[i])
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
var unquotedKeyPattern = regexp.MustCompile(`([{,]\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*:`)
|
||||
|
||||
// missingArrayBracketsPattern identifies a sequence of two or more JSON objects separated by commas
|
||||
// that immediately follow a colon, which indicates a missing array bracket `[` `]`.
|
||||
// E.g., "key": {"a": 1}, {"b": 2} -> "key": [{"a": 1}, {"b": 2}]
|
||||
// NOTE: The pattern uses (?:[^{}]|\{[^{}]*\})* to support single-level nested {} objects,
|
||||
// which handles cases like {"content": "x", "input": {"q": "y"}}
|
||||
var missingArrayBracketsPattern = regexp.MustCompile(`(:\s*)(\{(?:[^{}]|\{[^{}]*\})*\}(?:\s*,\s*\{(?:[^{}]|\{[^{}]*\})*\})+)`)
|
||||
|
||||
func RepairLooseJSON(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
// 1. Replace unquoted keys: {key: -> {"key":
|
||||
s = unquotedKeyPattern.ReplaceAllString(s, `$1"$2":`)
|
||||
|
||||
// 2. Heuristic: Fix missing array brackets for list of objects
|
||||
// e.g., : {obj1}, {obj2} -> : [{obj1}, {obj2}]
|
||||
// This specifically addresses DeepSeek's "list hallucination"
|
||||
s = missingArrayBracketsPattern.ReplaceAllString(s, `$1[$2]`)
|
||||
|
||||
return s
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var toolCallMarkupTagNames = []string{"tool_call", "function_call", "invoke"}
|
||||
var toolCallMarkupTagPatternByName = map[string]*regexp.Regexp{
|
||||
"tool_call": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?tool_call\b([^>]*)>(.*?)</(?:[a-z0-9_:-]+:)?tool_call>`),
|
||||
"function_call": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function_call\b([^>]*)>(.*?)</(?:[a-z0-9_:-]+:)?function_call>`),
|
||||
"invoke": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)>(.*?)</(?:[a-z0-9_:-]+:)?invoke>`),
|
||||
}
|
||||
var toolCallMarkupSelfClosingPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?invoke\b([^>]*)/>`)
|
||||
var toolCallMarkupKVPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)>`)
|
||||
var toolCallMarkupAttrPattern = regexp.MustCompile(`(?is)(name|function|tool)\s*=\s*"([^"]+)"`)
|
||||
var anyTagPattern = regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
var toolCallMarkupNameTagNames = []string{"name", "function"}
|
||||
var toolCallMarkupNamePatternByTag = map[string]*regexp.Regexp{
|
||||
"name": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?name\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?name>`),
|
||||
"function": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?function>`),
|
||||
}
|
||||
var toolCallMarkupArgsTagNames = []string{"input", "arguments", "argument", "parameters", "parameter", "args", "params"}
|
||||
var toolCallMarkupArgsPatternByTag = map[string]*regexp.Regexp{
|
||||
"input": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?input\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?input>`),
|
||||
"arguments": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?arguments\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?arguments>`),
|
||||
"argument": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?argument\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?argument>`),
|
||||
"parameters": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?parameters\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?parameters>`),
|
||||
"parameter": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?parameter\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?parameter>`),
|
||||
"args": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?args\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?args>`),
|
||||
"params": regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?params\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?params>`),
|
||||
}
|
||||
|
||||
func parseMarkupToolCalls(text string) []ParsedToolCall {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
if trimmed == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]ParsedToolCall, 0)
|
||||
for _, tagName := range toolCallMarkupTagNames {
|
||||
pattern := toolCallMarkupTagPatternByName[tagName]
|
||||
for _, m := range pattern.FindAllStringSubmatch(trimmed, -1) {
|
||||
if len(m) < 3 {
|
||||
continue
|
||||
}
|
||||
attrs := strings.TrimSpace(m[1])
|
||||
inner := strings.TrimSpace(m[2])
|
||||
if parsed := parseMarkupSingleToolCall(attrs, inner); parsed.Name != "" {
|
||||
out = append(out, parsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range toolCallMarkupSelfClosingPattern.FindAllStringSubmatch(trimmed, -1) {
|
||||
if len(m) < 2 {
|
||||
continue
|
||||
}
|
||||
if parsed := parseMarkupSingleToolCall(strings.TrimSpace(m[1]), ""); parsed.Name != "" {
|
||||
out = append(out, parsed)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseMarkupSingleToolCall(attrs string, inner string) ParsedToolCall {
|
||||
if parsed := parseToolCallsPayload(inner); len(parsed) > 0 {
|
||||
return parsed[0]
|
||||
}
|
||||
|
||||
name := ""
|
||||
if m := toolCallMarkupAttrPattern.FindStringSubmatch(attrs); len(m) >= 3 {
|
||||
name = strings.TrimSpace(m[2])
|
||||
}
|
||||
if name == "" {
|
||||
name = findMarkupTagValue(inner, toolCallMarkupNameTagNames, toolCallMarkupNamePatternByTag)
|
||||
}
|
||||
if name == "" {
|
||||
return ParsedToolCall{}
|
||||
}
|
||||
|
||||
input := map[string]any{}
|
||||
if argsRaw := findMarkupTagValue(inner, toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); argsRaw != "" {
|
||||
input = parseMarkupInput(argsRaw)
|
||||
} else if kv := parseMarkupKVObject(inner); len(kv) > 0 {
|
||||
input = kv
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}
|
||||
}
|
||||
|
||||
func parseMarkupInput(raw string) map[string]any {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return map[string]any{}
|
||||
}
|
||||
if parsed := parseToolCallInput(raw); len(parsed) > 0 {
|
||||
return parsed
|
||||
}
|
||||
if kv := parseMarkupKVObject(raw); len(kv) > 0 {
|
||||
return kv
|
||||
}
|
||||
return map[string]any{"_raw": stripTagText(raw)}
|
||||
}
|
||||
|
||||
func parseMarkupKVObject(text string) map[string]any {
|
||||
matches := toolCallMarkupKVPattern.FindAllStringSubmatch(strings.TrimSpace(text), -1)
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := map[string]any{}
|
||||
for _, m := range matches {
|
||||
if len(m) < 4 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(m[1])
|
||||
endKey := strings.TrimSpace(m[3])
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(key, endKey) {
|
||||
continue
|
||||
}
|
||||
value := strings.TrimSpace(stripTagText(m[2]))
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
var jsonValue any
|
||||
if json.Unmarshal([]byte(value), &jsonValue) == nil {
|
||||
out[key] = jsonValue
|
||||
continue
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stripTagText(text string) string {
|
||||
return strings.TrimSpace(anyTagPattern.ReplaceAllString(text, ""))
|
||||
}
|
||||
|
||||
func findMarkupTagValue(text string, tagNames []string, patternByTag map[string]*regexp.Regexp) string {
|
||||
for _, tag := range tagNames {
|
||||
pattern := patternByTag[tag]
|
||||
if pattern == nil {
|
||||
continue
|
||||
}
|
||||
if m := pattern.FindStringSubmatch(text); len(m) >= 2 {
|
||||
value := strings.TrimSpace(m[1])
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var toolNameLoosePattern = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
func resolveAllowedToolNameWithLooseMatch(name string, allowed map[string]struct{}, allowedCanonical map[string]string) string {
|
||||
if _, ok := allowed[name]; ok {
|
||||
return name
|
||||
}
|
||||
lower := strings.ToLower(strings.TrimSpace(name))
|
||||
if canonical, ok := allowedCanonical[lower]; ok {
|
||||
return canonical
|
||||
}
|
||||
if idx := strings.LastIndex(lower, "."); idx >= 0 && idx < len(lower)-1 {
|
||||
if canonical, ok := allowedCanonical[lower[idx+1:]]; ok {
|
||||
return canonical
|
||||
}
|
||||
}
|
||||
loose := toolNameLoosePattern.ReplaceAllString(lower, "")
|
||||
if loose == "" {
|
||||
return ""
|
||||
}
|
||||
for candidateLower, canonical := range allowedCanonical {
|
||||
if toolNameLoosePattern.ReplaceAllString(candidateLower, "") == loose {
|
||||
return canonical
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,232 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ParsedToolCall struct {
|
||||
Name string `json:"name"`
|
||||
Input map[string]any `json:"input"`
|
||||
}
|
||||
|
||||
type ToolCallParseResult struct {
|
||||
Calls []ParsedToolCall
|
||||
SawToolCallSyntax bool
|
||||
RejectedByPolicy bool
|
||||
RejectedToolNames []string
|
||||
}
|
||||
|
||||
func ParseToolCalls(text string, availableToolNames []string) []ParsedToolCall {
|
||||
return ParseToolCallsDetailed(text, availableToolNames).Calls
|
||||
}
|
||||
|
||||
func ParseToolCallsDetailed(text string, availableToolNames []string) ToolCallParseResult {
|
||||
result := ToolCallParseResult{}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return result
|
||||
}
|
||||
result.SawToolCallSyntax = looksLikeToolCallSyntax(text)
|
||||
if shouldSkipToolCallParsingForCodeFenceExample(text) {
|
||||
return result
|
||||
}
|
||||
|
||||
candidates := buildToolCallCandidates(text)
|
||||
for _, candidate := range candidates {
|
||||
if !isLikelyJSONToolPayloadCandidate(candidate) {
|
||||
continue
|
||||
}
|
||||
tc := parseToolCallsPayload(candidate)
|
||||
if len(tc) == 0 {
|
||||
continue
|
||||
}
|
||||
parsed := tc
|
||||
calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames)
|
||||
result.Calls = calls
|
||||
result.RejectedToolNames = rejectedNames
|
||||
result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0
|
||||
result.SawToolCallSyntax = true
|
||||
return result
|
||||
}
|
||||
var parsed []ParsedToolCall
|
||||
for _, candidate := range candidates {
|
||||
tc := parseXMLToolCalls(candidate)
|
||||
if len(tc) == 0 {
|
||||
tc = parseMarkupToolCalls(candidate)
|
||||
}
|
||||
if len(tc) == 0 {
|
||||
tc = parseToolCallsPayload(candidate)
|
||||
}
|
||||
if len(tc) == 0 {
|
||||
tc = parseTextKVToolCalls(candidate)
|
||||
}
|
||||
if len(tc) > 0 {
|
||||
parsed = tc
|
||||
result.SawToolCallSyntax = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseXMLToolCalls(text)
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseTextKVToolCalls(text)
|
||||
if len(parsed) == 0 {
|
||||
return result
|
||||
}
|
||||
}
|
||||
result.SawToolCallSyntax = true
|
||||
}
|
||||
|
||||
calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames)
|
||||
result.Calls = calls
|
||||
result.RejectedToolNames = rejectedNames
|
||||
result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0
|
||||
return result
|
||||
}
|
||||
func ParseStandaloneToolCalls(text string, availableToolNames []string) []ParsedToolCall {
|
||||
return ParseStandaloneToolCallsDetailed(text, availableToolNames).Calls
|
||||
}
|
||||
|
||||
func ParseStandaloneToolCallsDetailed(text string, availableToolNames []string) ToolCallParseResult {
|
||||
result := ToolCallParseResult{}
|
||||
trimmed := strings.TrimSpace(text)
|
||||
if trimmed == "" {
|
||||
return result
|
||||
}
|
||||
result.SawToolCallSyntax = looksLikeToolCallSyntax(trimmed)
|
||||
if shouldSkipToolCallParsingForCodeFenceExample(trimmed) {
|
||||
return result
|
||||
}
|
||||
candidates := buildToolCallCandidates(trimmed)
|
||||
var parsed []ParsedToolCall
|
||||
for _, candidate := range candidates {
|
||||
if !isLikelyJSONToolPayloadCandidate(candidate) {
|
||||
continue
|
||||
}
|
||||
parsed = parseToolCallsPayload(candidate)
|
||||
if len(parsed) == 0 {
|
||||
continue
|
||||
}
|
||||
result.SawToolCallSyntax = true
|
||||
calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames)
|
||||
result.Calls = calls
|
||||
result.RejectedToolNames = rejectedNames
|
||||
result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0
|
||||
return result
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
candidate = strings.TrimSpace(candidate)
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
parsed = parseXMLToolCalls(candidate)
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseMarkupToolCalls(candidate)
|
||||
}
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseToolCallsPayload(candidate)
|
||||
}
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseTextKVToolCalls(candidate)
|
||||
}
|
||||
if len(parsed) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseXMLToolCalls(trimmed)
|
||||
if len(parsed) == 0 {
|
||||
parsed = parseTextKVToolCalls(trimmed)
|
||||
if len(parsed) == 0 {
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
result.SawToolCallSyntax = true
|
||||
calls, rejectedNames := filterToolCallsDetailed(parsed, availableToolNames)
|
||||
result.Calls = calls
|
||||
result.RejectedToolNames = rejectedNames
|
||||
result.RejectedByPolicy = len(rejectedNames) > 0 && len(calls) == 0
|
||||
return result
|
||||
}
|
||||
|
||||
func filterToolCallsDetailed(parsed []ParsedToolCall, availableToolNames []string) ([]ParsedToolCall, []string) {
|
||||
out := make([]ParsedToolCall, 0, len(parsed))
|
||||
for _, tc := range parsed {
|
||||
if tc.Name == "" {
|
||||
continue
|
||||
}
|
||||
if tc.Input == nil {
|
||||
tc.Input = map[string]any{}
|
||||
}
|
||||
out = append(out, tc)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func resolveAllowedToolName(name string, allowed map[string]struct{}, allowedCanonical map[string]string) string {
|
||||
return resolveAllowedToolNameWithLooseMatch(name, allowed, allowedCanonical)
|
||||
}
|
||||
|
||||
func parseToolCallsPayload(payload string) []ParsedToolCall {
|
||||
var decoded any
|
||||
if err := json.Unmarshal([]byte(payload), &decoded); err != nil {
|
||||
// Try to repair backslashes first! Because LLMs often mix these two problems.
|
||||
repaired := repairInvalidJSONBackslashes(payload)
|
||||
// Try loose repair on top of that
|
||||
repaired = RepairLooseJSON(repaired)
|
||||
if err := json.Unmarshal([]byte(repaired), &decoded); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
switch v := decoded.(type) {
|
||||
case map[string]any:
|
||||
if tc, ok := v["tool_calls"]; ok {
|
||||
if isLikelyChatMessageEnvelope(v) {
|
||||
return nil
|
||||
}
|
||||
return parseToolCallList(tc)
|
||||
}
|
||||
if parsed, ok := parseToolCallItem(v); ok {
|
||||
return []ParsedToolCall{parsed}
|
||||
}
|
||||
case []any:
|
||||
return parseToolCallList(v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isLikelyChatMessageEnvelope(v map[string]any) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
if _, ok := v["tool_calls"]; !ok {
|
||||
return false
|
||||
}
|
||||
if role, ok := v["role"].(string); ok {
|
||||
switch strings.ToLower(strings.TrimSpace(role)) {
|
||||
case "assistant", "tool", "user", "system":
|
||||
return true
|
||||
}
|
||||
}
|
||||
if _, ok := v["tool_call_id"]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := v["content"]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func looksLikeToolCallSyntax(text string) bool {
|
||||
lower := strings.ToLower(text)
|
||||
return strings.Contains(lower, "tool_calls") ||
|
||||
strings.Contains(lower, "\"function\"") ||
|
||||
strings.Contains(lower, "functioncall") ||
|
||||
strings.Contains(lower, "\"tool_use\"") ||
|
||||
strings.Contains(lower, "<tool_call") ||
|
||||
strings.Contains(lower, "<function_call") ||
|
||||
strings.Contains(lower, "<function_name") ||
|
||||
strings.Contains(lower, "<invoke") ||
|
||||
strings.Contains(lower, "function.name:")
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package util
|
||||
|
||||
import "strings"
|
||||
|
||||
func isLikelyJSONToolPayloadCandidate(candidate string) bool {
|
||||
trimmed := strings.TrimSpace(candidate)
|
||||
if trimmed == "" {
|
||||
return false
|
||||
}
|
||||
if !(strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[")) {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(trimmed)
|
||||
return strings.Contains(lower, "tool_calls") ||
|
||||
strings.Contains(lower, "\"function\"") ||
|
||||
strings.Contains(lower, "functioncall") ||
|
||||
strings.Contains(lower, "\"tool_use\"")
|
||||
}
|
||||
|
||||
func parseToolCallList(v any) []ParsedToolCall {
|
||||
items, ok := v.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := make([]ParsedToolCall, 0, len(items))
|
||||
for _, item := range items {
|
||||
m, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if tc, ok := parseToolCallItem(m); ok {
|
||||
out = append(out, tc)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseToolCallItem(m map[string]any) (ParsedToolCall, bool) {
|
||||
name, _ := m["name"].(string)
|
||||
inputRaw, hasInput := m["input"]
|
||||
if fnCall, ok := m["functionCall"].(map[string]any); ok {
|
||||
if name == "" {
|
||||
name, _ = fnCall["name"].(string)
|
||||
}
|
||||
if !hasInput {
|
||||
if v, ok := fnCall["args"]; ok {
|
||||
inputRaw = v
|
||||
hasInput = true
|
||||
}
|
||||
}
|
||||
if !hasInput {
|
||||
if v, ok := fnCall["arguments"]; ok {
|
||||
inputRaw = v
|
||||
hasInput = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if fn, ok := m["function"].(map[string]any); ok {
|
||||
if name == "" {
|
||||
name, _ = fn["name"].(string)
|
||||
}
|
||||
if !hasInput {
|
||||
if v, ok := fn["arguments"]; ok {
|
||||
inputRaw = v
|
||||
hasInput = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !hasInput {
|
||||
for _, key := range []string{"arguments", "args", "parameters", "params"} {
|
||||
if v, ok := m[key]; ok {
|
||||
inputRaw = v
|
||||
hasInput = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
return ParsedToolCall{
|
||||
Name: strings.TrimSpace(name),
|
||||
Input: parseToolCallInput(inputRaw),
|
||||
}, true
|
||||
}
|
||||
@@ -1,457 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var xmlToolCallPattern = regexp.MustCompile(`(?is)<tool_call>\s*(.*?)\s*</tool_call>`)
|
||||
var functionCallPattern = regexp.MustCompile(`(?is)<function_call>\s*([^<]+?)\s*</function_call>`)
|
||||
var functionParamPattern = regexp.MustCompile(`(?is)<function\s+parameter\s+name="([^"]+)"\s*>\s*(.*?)\s*</function\s+parameter>`)
|
||||
var antmlFunctionCallPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?function_call[^>]*(?:name|function)="([^"]+)"[^>]*>\s*(.*?)\s*</(?:[a-z0-9_]+:)?function_call>`)
|
||||
var antmlArgumentPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?argument\s+name="([^"]+)"\s*>\s*(.*?)\s*</(?:[a-z0-9_]+:)?argument>`)
|
||||
var antmlParametersPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_]+:)?parameters\s*>\s*(\{.*?\})\s*</(?:[a-z0-9_]+:)?parameters>`)
|
||||
var invokeCallPattern = regexp.MustCompile(`(?is)<invoke\s+name="([^"]+)"\s*>(.*?)</invoke>`)
|
||||
var invokeParamPattern = regexp.MustCompile(`(?is)<parameter\s+name="([^"]+)"\s*>\s*(.*?)\s*</parameter>`)
|
||||
var toolUseFunctionPattern = regexp.MustCompile(`(?is)<tool_use>\s*<function\s+name="([^"]+)"\s*>(.*?)</function>\s*</tool_use>`)
|
||||
var toolUseNameParametersPattern = regexp.MustCompile(`(?is)<tool_use>\s*<tool_name>\s*([^<]+?)\s*</tool_name>\s*<parameters>\s*(.*?)\s*</parameters>\s*</tool_use>`)
|
||||
var toolUseFunctionNameParametersPattern = regexp.MustCompile(`(?is)<tool_use>\s*<function_name>\s*([^<]+?)\s*</function_name>\s*<parameters>\s*(.*?)\s*</parameters>\s*</tool_use>`)
|
||||
var toolUseToolNameBodyPattern = regexp.MustCompile(`(?is)<tool_use>\s*<tool_name>\s*([^<]+?)\s*</tool_name>\s*(.*?)\s*</tool_use>`)
|
||||
var xmlToolNamePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?tool_name\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?tool_name>`),
|
||||
regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?function_name\b[^>]*>(.*?)</(?:[a-z0-9_:-]+:)?function_name>`),
|
||||
}
|
||||
|
||||
func parseXMLToolCalls(text string) []ParsedToolCall {
|
||||
matches := xmlToolCallPattern.FindAllString(text, -1)
|
||||
out := make([]ParsedToolCall, 0, len(matches)+1)
|
||||
for _, block := range matches {
|
||||
call, ok := parseSingleXMLToolCall(block)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, call)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
if call, ok := parseFunctionCallTagStyle(text); ok {
|
||||
return []ParsedToolCall{call}
|
||||
}
|
||||
if calls := parseAntmlFunctionCallStyles(text); len(calls) > 0 {
|
||||
return calls
|
||||
}
|
||||
if call, ok := parseInvokeFunctionCallStyle(text); ok {
|
||||
return []ParsedToolCall{call}
|
||||
}
|
||||
if call, ok := parseToolUseFunctionStyle(text); ok {
|
||||
return []ParsedToolCall{call}
|
||||
}
|
||||
if call, ok := parseToolUseNameParametersStyle(text); ok {
|
||||
return []ParsedToolCall{call}
|
||||
}
|
||||
if call, ok := parseToolUseFunctionNameParametersStyle(text); ok {
|
||||
return []ParsedToolCall{call}
|
||||
}
|
||||
if call, ok := parseToolUseToolNameBodyStyle(text); ok {
|
||||
return []ParsedToolCall{call}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseSingleXMLToolCall(block string) (ParsedToolCall, bool) {
|
||||
inner := strings.TrimSpace(block)
|
||||
inner = strings.TrimPrefix(inner, "<tool_call>")
|
||||
inner = strings.TrimSuffix(inner, "</tool_call>")
|
||||
inner = strings.TrimSpace(inner)
|
||||
if strings.HasPrefix(inner, "{") {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal([]byte(inner), &payload); err == nil {
|
||||
name := strings.TrimSpace(asString(payload["tool"]))
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(asString(payload["tool_name"]))
|
||||
}
|
||||
if name != "" {
|
||||
input := map[string]any{}
|
||||
if params, ok := payload["params"].(map[string]any); ok {
|
||||
input = params
|
||||
} else if params, ok := payload["parameters"].(map[string]any); ok {
|
||||
input = params
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
name := ""
|
||||
params := extractXMLToolParamsByRegex(inner)
|
||||
dec := xml.NewDecoder(strings.NewReader(block))
|
||||
inParams := false
|
||||
inTool := false
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
tag := strings.ToLower(t.Name.Local)
|
||||
switch tag {
|
||||
case "tool":
|
||||
inTool = true
|
||||
for _, attr := range t.Attr {
|
||||
if strings.EqualFold(strings.TrimSpace(attr.Name.Local), "name") && strings.TrimSpace(name) == "" {
|
||||
name = strings.TrimSpace(attr.Value)
|
||||
}
|
||||
}
|
||||
case "parameters":
|
||||
inParams = true
|
||||
var node struct {
|
||||
Inner string `xml:",innerxml"`
|
||||
}
|
||||
if err := dec.DecodeElement(&node, &t); err == nil {
|
||||
inner := strings.TrimSpace(node.Inner)
|
||||
if inner != "" {
|
||||
if parsed := parseToolCallInput(inner); len(parsed) > 0 {
|
||||
if len(parsed) == 1 {
|
||||
if _, onlyRaw := parsed["_raw"]; onlyRaw {
|
||||
if kv := parseMarkupKVObject(inner); len(kv) > 0 {
|
||||
for k, vv := range kv {
|
||||
params[k] = vv
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, vv := range parsed {
|
||||
params[k] = vv
|
||||
}
|
||||
} else if kv := parseMarkupKVObject(inner); len(kv) > 0 {
|
||||
for k, vv := range kv {
|
||||
params[k] = vv
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
inParams = false
|
||||
case "tool_name", "function_name", "name":
|
||||
var v string
|
||||
if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" {
|
||||
if inParams {
|
||||
params[t.Name.Local] = strings.TrimSpace(v)
|
||||
break
|
||||
}
|
||||
name = strings.TrimSpace(v)
|
||||
}
|
||||
case "input", "arguments", "argument", "args", "params":
|
||||
var v string
|
||||
if err := dec.DecodeElement(&v, &t); err == nil && strings.TrimSpace(v) != "" {
|
||||
if parsed := parseToolCallInput(strings.TrimSpace(v)); len(parsed) > 0 {
|
||||
for k, vv := range parsed {
|
||||
params[k] = vv
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
if inParams || inTool {
|
||||
var v string
|
||||
if err := dec.DecodeElement(&v, &t); err == nil {
|
||||
params[t.Name.Local] = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
case xml.EndElement:
|
||||
tag := strings.ToLower(t.Name.Local)
|
||||
if tag == "parameters" {
|
||||
inParams = false
|
||||
}
|
||||
if tag == "tool" {
|
||||
inTool = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(name) == "" {
|
||||
name = strings.TrimSpace(extractXMLToolNameByRegex(stripTopLevelXMLParameters(inner)))
|
||||
}
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
return ParsedToolCall{Name: strings.TrimSpace(name), Input: params}, true
|
||||
}
|
||||
|
||||
func stripTopLevelXMLParameters(inner string) string {
|
||||
out := strings.TrimSpace(inner)
|
||||
for {
|
||||
idx := strings.Index(strings.ToLower(out), "<parameters")
|
||||
if idx < 0 {
|
||||
return out
|
||||
}
|
||||
segment := out[idx:]
|
||||
segmentLower := strings.ToLower(segment)
|
||||
openEnd := strings.Index(segmentLower, ">")
|
||||
if openEnd < 0 {
|
||||
return out
|
||||
}
|
||||
closeIdx := strings.Index(segmentLower, "</parameters>")
|
||||
if closeIdx < 0 {
|
||||
return out[:idx]
|
||||
}
|
||||
end := idx + closeIdx + len("</parameters>")
|
||||
out = out[:idx] + out[end:]
|
||||
}
|
||||
}
|
||||
|
||||
func extractXMLToolNameByRegex(inner string) string {
|
||||
for _, pattern := range xmlToolNamePatterns {
|
||||
if m := pattern.FindStringSubmatch(inner); len(m) >= 2 {
|
||||
if v := strings.TrimSpace(stripTagText(m[1])); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractXMLToolParamsByRegex(inner string) map[string]any {
|
||||
raw := findMarkupTagValue(inner, toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag)
|
||||
if raw == "" {
|
||||
return map[string]any{}
|
||||
}
|
||||
parsed := parseMarkupInput(raw)
|
||||
if parsed == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func parseFunctionCallTagStyle(text string) (ParsedToolCall, bool) {
|
||||
m := functionCallPattern.FindStringSubmatch(text)
|
||||
if len(m) < 2 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
input := map[string]any{}
|
||||
for _, pm := range functionParamPattern.FindAllStringSubmatch(text, -1) {
|
||||
if len(pm) < 3 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(pm[1])
|
||||
val := strings.TrimSpace(pm[2])
|
||||
if key != "" {
|
||||
input[key] = val
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseAntmlFunctionCallStyles(text string) []ParsedToolCall {
|
||||
matches := antmlFunctionCallPattern.FindAllStringSubmatch(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ParsedToolCall, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
if call, ok := parseSingleAntmlFunctionCallMatch(m); ok {
|
||||
out = append(out, call)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseSingleAntmlFunctionCallMatch(m []string) (ParsedToolCall, bool) {
|
||||
if len(m) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
body := strings.TrimSpace(m[2])
|
||||
input := map[string]any{}
|
||||
if strings.HasPrefix(body, "{") {
|
||||
if err := json.Unmarshal([]byte(body), &input); err == nil {
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
}
|
||||
if pm := antmlParametersPattern.FindStringSubmatch(body); len(pm) >= 2 {
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(pm[1])), &input); err == nil {
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
}
|
||||
for _, am := range antmlArgumentPattern.FindAllStringSubmatch(body, -1) {
|
||||
if len(am) < 3 {
|
||||
continue
|
||||
}
|
||||
k := strings.TrimSpace(am[1])
|
||||
v := strings.TrimSpace(am[2])
|
||||
if k != "" {
|
||||
input[k] = v
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseInvokeFunctionCallStyle(text string) (ParsedToolCall, bool) {
|
||||
m := invokeCallPattern.FindStringSubmatch(text)
|
||||
if len(m) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
input := map[string]any{}
|
||||
for _, pm := range invokeParamPattern.FindAllStringSubmatch(m[2], -1) {
|
||||
if len(pm) < 3 {
|
||||
continue
|
||||
}
|
||||
k := strings.TrimSpace(pm[1])
|
||||
v := strings.TrimSpace(pm[2])
|
||||
if k != "" {
|
||||
input[k] = v
|
||||
}
|
||||
}
|
||||
if len(input) == 0 {
|
||||
if argsRaw := findMarkupTagValue(m[2], toolCallMarkupArgsTagNames, toolCallMarkupArgsPatternByTag); argsRaw != "" {
|
||||
input = parseMarkupInput(argsRaw)
|
||||
} else if kv := parseMarkupKVObject(m[2]); len(kv) > 0 {
|
||||
input = kv
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseToolUseFunctionStyle(text string) (ParsedToolCall, bool) {
|
||||
m := toolUseFunctionPattern.FindStringSubmatch(text)
|
||||
if len(m) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
body := m[2]
|
||||
input := map[string]any{}
|
||||
for _, pm := range invokeParamPattern.FindAllStringSubmatch(body, -1) {
|
||||
if len(pm) < 3 {
|
||||
continue
|
||||
}
|
||||
k := strings.TrimSpace(pm[1])
|
||||
v := strings.TrimSpace(pm[2])
|
||||
if k != "" {
|
||||
input[k] = v
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseToolUseNameParametersStyle(text string) (ParsedToolCall, bool) {
|
||||
m := toolUseNameParametersPattern.FindStringSubmatch(text)
|
||||
if len(m) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
raw := strings.TrimSpace(m[2])
|
||||
input := map[string]any{}
|
||||
if raw != "" {
|
||||
if parsed := parseToolCallInput(raw); len(parsed) > 0 {
|
||||
input = parsed
|
||||
} else if kv := parseMarkupKVObject(raw); len(kv) > 0 {
|
||||
input = kv
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseToolUseFunctionNameParametersStyle(text string) (ParsedToolCall, bool) {
|
||||
m := toolUseFunctionNameParametersPattern.FindStringSubmatch(text)
|
||||
if len(m) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
raw := strings.TrimSpace(m[2])
|
||||
input := map[string]any{}
|
||||
if raw != "" {
|
||||
if parsed := parseToolCallInput(raw); len(parsed) > 0 {
|
||||
input = parsed
|
||||
} else if kv := parseMarkupKVObject(raw); len(kv) > 0 {
|
||||
input = kv
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseToolUseToolNameBodyStyle(text string) (ParsedToolCall, bool) {
|
||||
m := toolUseToolNameBodyPattern.FindStringSubmatch(text)
|
||||
if len(m) < 3 {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
name := strings.TrimSpace(m[1])
|
||||
if name == "" {
|
||||
return ParsedToolCall{}, false
|
||||
}
|
||||
body := strings.TrimSpace(m[2])
|
||||
input := map[string]any{}
|
||||
if body != "" {
|
||||
if kv := parseXMLChildKV(body); len(kv) > 0 {
|
||||
input = kv
|
||||
} else if kv := parseMarkupKVObject(body); len(kv) > 0 {
|
||||
input = kv
|
||||
} else if parsed := parseToolCallInput(body); len(parsed) > 0 {
|
||||
input = parsed
|
||||
}
|
||||
}
|
||||
return ParsedToolCall{Name: name, Input: input}, true
|
||||
}
|
||||
|
||||
func parseXMLChildKV(body string) map[string]any {
|
||||
trimmed := strings.TrimSpace(body)
|
||||
if trimmed == "" {
|
||||
return nil
|
||||
}
|
||||
dec := xml.NewDecoder(strings.NewReader("<root>" + trimmed + "</root>"))
|
||||
out := map[string]any{}
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
start, ok := tok.(xml.StartElement)
|
||||
if !ok || strings.EqualFold(start.Name.Local, "root") {
|
||||
continue
|
||||
}
|
||||
var v string
|
||||
if err := dec.DecodeElement(&v, &start); err != nil {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(start.Name.Local)
|
||||
val := strings.TrimSpace(v)
|
||||
if key == "" || val == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = val
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func asString(v any) string {
|
||||
s, _ := v.(string)
|
||||
return s
|
||||
}
|
||||
@@ -1,693 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseToolCalls(t *testing.T) {
|
||||
text := `prefix {"tool_calls":[{"name":"search","input":{"q":"golang"}}]} suffix`
|
||||
calls := ParseToolCalls(text, []string{"search"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Name != "search" {
|
||||
t.Fatalf("unexpected tool name: %s", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["q"] != "golang" {
|
||||
t.Fatalf("unexpected args: %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsIgnoresFencedJSON(t *testing.T) {
|
||||
text := "I will call tools now\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"news\"}}]}\n```"
|
||||
calls := ParseToolCalls(text, []string{"search"})
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("expected fenced tool_call payload to be ignored, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsWithFunctionArgumentsString(t *testing.T) {
|
||||
text := `{"tool_calls":[{"function":{"name":"get_weather","arguments":"{\"city\":\"beijing\"}"}}]}`
|
||||
calls := ParseToolCalls(text, []string{"get_weather"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Name != "get_weather" {
|
||||
t.Fatalf("unexpected tool name: %s", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["city"] != "beijing" {
|
||||
t.Fatalf("unexpected args: %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsKeepsUnknownToolName(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"unknown","input":{}}]}`
|
||||
calls := ParseToolCalls(text, []string{"search"})
|
||||
if len(calls) != 1 || calls[0].Name != "unknown" {
|
||||
t.Fatalf("expected unknown tool to be preserved, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsKeepsOriginalToolNameCase(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"Bash","input":{"command":"ls -al"}}]}`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsDetailedDoesNotRejectByPolicy(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"unknown","input":{}}]}`
|
||||
res := ParseToolCallsDetailed(text, []string{"search"})
|
||||
if !res.SawToolCallSyntax {
|
||||
t.Fatalf("expected SawToolCallSyntax=true, got %#v", res)
|
||||
}
|
||||
if res.RejectedByPolicy {
|
||||
t.Fatalf("expected RejectedByPolicy=false, got %#v", res)
|
||||
}
|
||||
if len(res.Calls) != 1 || res.Calls[0].Name != "unknown" {
|
||||
t.Fatalf("expected call to be preserved, got %#v", res.Calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsDetailedAllowsWhenAllowListEmpty(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}`
|
||||
res := ParseToolCallsDetailed(text, nil)
|
||||
if !res.SawToolCallSyntax {
|
||||
t.Fatalf("expected SawToolCallSyntax=true, got %#v", res)
|
||||
}
|
||||
if res.RejectedByPolicy {
|
||||
t.Fatalf("expected RejectedByPolicy=false, got %#v", res)
|
||||
}
|
||||
if len(res.Calls) != 1 || res.Calls[0].Name != "search" {
|
||||
t.Fatalf("expected calls when allow-list is empty, got %#v", res.Calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatOpenAIToolCalls(t *testing.T) {
|
||||
formatted := FormatOpenAIToolCalls([]ParsedToolCall{{Name: "search", Input: map[string]any{"q": "x"}}})
|
||||
if len(formatted) != 1 {
|
||||
t.Fatalf("expected 1, got %d", len(formatted))
|
||||
}
|
||||
fn, _ := formatted[0]["function"].(map[string]any)
|
||||
if fn["name"] != "search" {
|
||||
t.Fatalf("unexpected function name: %#v", fn)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStandaloneToolCallsSupportsMixedProsePayload(t *testing.T) {
|
||||
mixed := `这里是示例：{"tool_calls":[{"name":"search","input":{"q":"go"}}]}`
|
||||
if calls := ParseStandaloneToolCalls(mixed, []string{"search"}); len(calls) != 1 {
|
||||
t.Fatalf("expected standalone parser to parse mixed prose payload, got %#v", calls)
|
||||
}
|
||||
|
||||
standalone := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}`
|
||||
calls := ParseStandaloneToolCalls(standalone, []string{"search"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected standalone parser to match, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStandaloneToolCallsIgnoresFencedCodeBlock(t *testing.T) {
|
||||
fenced := "```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```"
|
||||
if calls := ParseStandaloneToolCalls(fenced, []string{"search"}); len(calls) != 0 {
|
||||
t.Fatalf("expected fenced tool_call payload to be ignored, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStandaloneToolCallsIgnoresChatTranscriptEnvelope(t *testing.T) {
|
||||
transcript := `[{"role":"user","content":"请展示完整会话"},{"role":"assistant","content":null,"tool_calls":[{"function":{"name":"search","arguments":"{\"q\":\"go\"}"}}]}]`
|
||||
if calls := ParseStandaloneToolCalls(transcript, []string{"search"}); len(calls) != 0 {
|
||||
t.Fatalf("expected transcript envelope not to trigger tool call parse, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsAllowsQualifiedToolName(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"mcp.search_web","input":{"q":"golang"}}]}`
|
||||
calls := ParseToolCalls(text, []string{"search_web"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "mcp.search_web" {
|
||||
t.Fatalf("expected original tool name mcp.search_web, got %q", calls[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsAllowsPunctuationVariantToolName(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"read-file","input":{"path":"README.md"}}]}`
|
||||
calls := ParseToolCalls(text, []string{"read_file"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "read-file" {
|
||||
t.Fatalf("expected original tool name read-file, got %q", calls[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsClaudeXMLToolCall(t *testing.T) {
|
||||
text := `<tool_call><tool_name>Bash</tool_name><parameters><command>pwd</command><description>show cwd</description></parameters></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsCanonicalXMLParametersJSON(t *testing.T) {
|
||||
text := `<tool_call><tool_name>get_weather</tool_name><parameters>{"city":"beijing","unit":"c"}</parameters></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"get_weather"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "get_weather" {
|
||||
t.Fatalf("expected tool name get_weather, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["city"] != "beijing" || calls[0].Input["unit"] != "c" {
|
||||
t.Fatalf("expected parsed json parameters, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsPreservesRawMalformedXMLParameters(t *testing.T) {
|
||||
text := `<tool_call><tool_name>execute_command</tool_name><parameters>cd /root && git status</parameters></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"execute_command"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "execute_command" {
|
||||
t.Fatalf("expected tool name execute_command, got %q", calls[0].Name)
|
||||
}
|
||||
raw, ok := calls[0].Input["_raw"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("expected raw argument tracking, got %#v", calls[0].Input)
|
||||
}
|
||||
if raw != "cd /root && git status" {
|
||||
t.Fatalf("expected raw arguments to be preserved, got %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsXMLParametersJSONWithAmpersandCommand(t *testing.T) {
|
||||
text := `<tool_calls><tool_call><tool_name>execute_command</tool_name><parameters>{"command":"sshpass -p 'xxx' ssh -o StrictHostKeyChecking=no -p 1111 root@111.111.111.111 'cd /root && git clone https://github.com/ericc-ch/copilot-api.git'","cwd":null,"timeout":null}</parameters></tool_call></tool_calls>`
|
||||
calls := ParseToolCalls(text, []string{"execute_command"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "execute_command" {
|
||||
t.Fatalf("expected tool name execute_command, got %q", calls[0].Name)
|
||||
}
|
||||
cmd, _ := calls[0].Input["command"].(string)
|
||||
if !strings.Contains(cmd, "&& git clone") {
|
||||
t.Fatalf("expected command to keep && segment, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsDoesNotTreatParameterNameTagAsToolName(t *testing.T) {
|
||||
text := `<tool_call><tool name="execute_command"><parameters><name>file.txt</name><command>pwd</command></parameters></tool></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"execute_command"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "execute_command" {
|
||||
t.Fatalf("expected tool name execute_command, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["name"] != "file.txt" {
|
||||
t.Fatalf("expected parameter name preserved, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsPrefersJSONPayloadOverIncidentalXMLInString(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"search","input":{"q":"latest <tool_call><tool_name>wrong</tool_name><parameters>{\"x\":1}</parameters></tool_call>"}}]}`
|
||||
calls := ParseToolCallsDetailed(text, []string{"search"}).Calls
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "search" {
|
||||
t.Fatalf("expected tool name search, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["q"] == nil {
|
||||
t.Fatalf("expected q argument from json payload, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsDetailedMarksXMLToolCallSyntax(t *testing.T) {
|
||||
text := `<tool_call><tool_name>Bash</tool_name><parameters><command>pwd</command></parameters></tool_call>`
|
||||
res := ParseToolCallsDetailed(text, []string{"bash"})
|
||||
if !res.SawToolCallSyntax {
|
||||
t.Fatalf("expected SawToolCallSyntax=true, got %#v", res)
|
||||
}
|
||||
if len(res.Calls) != 1 {
|
||||
t.Fatalf("expected one parsed call, got %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsClaudeXMLJSONToolCall(t *testing.T) {
|
||||
text := `<tool_call>{"tool":"Bash","params":{"command":"pwd","description":"show cwd"}}</tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsFunctionCallTagStyle(t *testing.T) {
|
||||
text := `<function_call>Bash</function_call><function parameter name="command">ls -la</function parameter><function parameter name="description">list</function parameter>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "ls -la" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsAntmlFunctionCallStyle(t *testing.T) {
|
||||
text := `<antml:function_calls><antml:function_call name="Bash">{"command":"pwd","description":"x"}</antml:function_call></antml:function_calls>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsAntmlArgumentStyle(t *testing.T) {
|
||||
text := `<antml:function_calls><antml:function_call id="1" name="Bash"><antml:argument name="command">pwd</antml:argument><antml:argument name="description">x</antml:argument></antml:function_call></antml:function_calls>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsInvokeFunctionCallStyle(t *testing.T) {
|
||||
text := `<function_calls><invoke name="Bash"><parameter name="command">pwd</parameter><parameter name="description">d</parameter></invoke></function_calls>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsGeminiFunctionCallJSON(t *testing.T) {
|
||||
text := `{"functionCall":{"name":"search_web","args":{"query":"latest"}}}`
|
||||
calls := ParseToolCalls(text, []string{"search_web"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "search_web" {
|
||||
t.Fatalf("expected search_web, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["query"] != "latest" {
|
||||
t.Fatalf("expected query argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsClaudeToolUseJSON(t *testing.T) {
|
||||
text := `{"type":"tool_use","name":"read_file","input":{"path":"README.md"}}`
|
||||
calls := ParseToolCalls(text, []string{"read_file"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "read_file" {
|
||||
t.Fatalf("expected read_file, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["path"] != "README.md" {
|
||||
t.Fatalf("expected path argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsToolUseFunctionParameterStyle(t *testing.T) {
|
||||
text := `<tool_use><function name="search_web"><parameter name="query">test</parameter></function></tool_use>`
|
||||
calls := ParseToolCalls(text, []string{"search_web"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "search_web" {
|
||||
t.Fatalf("expected canonical tool name search_web, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["query"] != "test" {
|
||||
t.Fatalf("expected query argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsToolUseNameParametersStyle(t *testing.T) {
|
||||
text := `<tool_use><tool_name>write_file</tool_name><parameters>{"path":"/tmp/a.txt","content":"abc"}</parameters></tool_use>`
|
||||
calls := ParseToolCalls(text, []string{"write_file"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "write_file" {
|
||||
t.Fatalf("expected tool name write_file, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["path"] != "/tmp/a.txt" {
|
||||
t.Fatalf("expected path argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsToolUseFunctionNameParametersStyle(t *testing.T) {
|
||||
text := `<tool_use><function_name>write_file</function_name><parameters>{"path":"/tmp/b.txt","content":"xyz"}</parameters></tool_use>`
|
||||
calls := ParseToolCalls(text, []string{"write_file"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "write_file" {
|
||||
t.Fatalf("expected tool name write_file, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["content"] != "xyz" {
|
||||
t.Fatalf("expected content argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsToolUseToolNameBodyStyle(t *testing.T) {
|
||||
text := `<tool_use><tool_name>write_file</tool_name><path>/tmp/c.txt</path><content>hello</content></tool_use>`
|
||||
calls := ParseToolCalls(text, []string{"write_file"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "write_file" {
|
||||
t.Fatalf("expected tool name write_file, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["path"] != "/tmp/c.txt" {
|
||||
t.Fatalf("expected path argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsNestedToolTagStyle(t *testing.T) {
|
||||
text := `<tool_call><tool name="Bash"><command>pwd</command><description>show cwd</description></tool></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsAntmlFunctionAttributeWithParametersTag(t *testing.T) {
|
||||
text := `<antml:function_calls><antml:function_call id="x" function="Bash"><antml:parameters>{"command":"pwd"}</antml:parameters></antml:function_call></antml:function_calls>`
|
||||
calls := ParseToolCalls(text, []string{"bash"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" {
|
||||
t.Fatalf("expected original tool name Bash, got %q", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "pwd" {
|
||||
t.Fatalf("expected command argument, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsSupportsMultipleAntmlFunctionCalls(t *testing.T) {
|
||||
text := `<antml:function_calls><antml:function_call id="1" function="Bash"><antml:parameters>{"command":"pwd"}</antml:parameters></antml:function_call><antml:function_call id="2" function="Read"><antml:parameters>{"file_path":"README.md"}</antml:parameters></antml:function_call></antml:function_calls>`
|
||||
calls := ParseToolCalls(text, []string{"bash", "read"})
|
||||
if len(calls) != 2 {
|
||||
t.Fatalf("expected 2 calls, got %#v", calls)
|
||||
}
|
||||
if calls[0].Name != "Bash" || calls[1].Name != "Read" {
|
||||
t.Fatalf("expected original names [Bash Read], got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsDoesNotAcceptMismatchedMarkupTags(t *testing.T) {
|
||||
text := `<tool_call><name>read_file</function><arguments>{"path":"README.md"}</arguments></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"read_file"})
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("expected mismatched tags to be rejected, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsDoesNotTreatParametersFunctionNameAsToolName(t *testing.T) {
|
||||
text := `<tool_call><parameters><function_name>data_only</function_name><path>README.md</path></parameters></tool_call>`
|
||||
calls := ParseToolCalls(text, []string{"read_file"})
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("expected no tool call when function_name appears only under parameters, got %#v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairInvalidJSONBackslashes(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{`{"path": "C:\Users\name"}`, `{"path": "C:\\Users\name"}`},
|
||||
{`{"cmd": "cd D:\git_codes"}`, `{"cmd": "cd D:\\git_codes"}`},
|
||||
{`{"text": "line1\nline2"}`, `{"text": "line1\nline2"}`},
|
||||
{`{"path": "D:\\back\\slash"}`, `{"path": "D:\\back\\slash"}`},
|
||||
{`{"unicode": "\u2705"}`, `{"unicode": "\u2705"}`},
|
||||
{`{"invalid_u": "\u123"}`, `{"invalid_u": "\\u123"}`},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := repairInvalidJSONBackslashes(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("repairInvalidJSONBackslashes(%s) = %s; want %s", tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairLooseJSON(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{`{tool_calls: [{"name": "search", "input": {"q": "go"}}]}`, `{"tool_calls": [{"name": "search", "input": {"q": "go"}}]}`},
|
||||
{`{name: "search", input: {q: "go"}}`, `{"name": "search", "input": {"q": "go"}}`},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := RepairLooseJSON(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("RepairLooseJSON(%s) = %s; want %s", tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsWithUnquotedKeys(t *testing.T) {
|
||||
text := `这里是列表：{tool_calls: [{"name": "todowrite", "input": {"todos": "test"}}]}`
|
||||
availableTools := []string{"todowrite"}
|
||||
|
||||
parsed := ParseToolCalls(text, availableTools)
|
||||
if len(parsed) != 1 {
|
||||
t.Fatalf("expected 1 tool call, got %d", len(parsed))
|
||||
}
|
||||
if parsed[0].Name != "todowrite" {
|
||||
t.Errorf("expected tool todowrite, got %s", parsed[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsWithInvalidBackslashes(t *testing.T) {
|
||||
// DeepSeek sometimes outputs Windows paths with single backslashes in JSON strings
|
||||
// Note: using raw string to simulate what AI actually sends in the stream
|
||||
text := `好的，执行以下命令：{"name": "execute_command", "input": "{\"command\": \"cd D:\git_codes && dir\"}"}`
|
||||
availableTools := []string{"execute_command"}
|
||||
|
||||
parsed := ParseToolCalls(text, availableTools)
|
||||
// If standard JSON fails, buildToolCallCandidates should still extract the object,
|
||||
// and parseToolCallsPayload should repair it.
|
||||
if len(parsed) != 1 {
|
||||
// If it still fails, let's see why
|
||||
candidates := buildToolCallCandidates(text)
|
||||
t.Logf("Candidates: %v", candidates)
|
||||
t.Fatalf("expected 1 tool call, got %d", len(parsed))
|
||||
}
|
||||
|
||||
cmd, ok := parsed[0].Input["command"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("expected command string in input, got %v", parsed[0].Input)
|
||||
}
|
||||
|
||||
expected := "cd D:\\git_codes && dir"
|
||||
if cmd != expected {
|
||||
t.Errorf("expected command %q, got %q", expected, cmd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsWithDeepSeekHallucination(t *testing.T) {
|
||||
// 模拟 DeepSeek 典型的幻觉输出：未加引号的键名 + 包含 Windows 路径的嵌套 JSON 字符串 + 漏掉列表的方括号
|
||||
text := `检测到实施意图——实现经典算法。需在misc/目录创建Python文件。
|
||||
关键约束:
|
||||
1. Windows UTF-8编码处理
|
||||
2. 必须用绝对路径导入
|
||||
3. 禁止write覆盖已有文件（misc/目录允许创建新文件）
|
||||
将任务分解并委托：
|
||||
- 研究8皇后算法模式（并行探索）
|
||||
- 实现带可视化输出的解决方案（unspecified-high）
|
||||
先创建todo列表追踪步骤。
|
||||
{tool_calls: [{"name": "todowrite", "input": {"todos": {"content": "研究8皇后问题算法模式（回溯法）和输出格式", "status": "pending", "priority": "high"}, {"content": "在misc/目录创建8皇后Python脚本，包含完整解决方案和可视化输出", "status": "pending", "priority": "high"}, {"content": "验证脚本正确性（运行测试）", "status": "pending", "priority": "medium"}}}]}`
|
||||
|
||||
availableTools := []string{"todowrite"}
|
||||
parsed := ParseToolCalls(text, availableTools)
|
||||
|
||||
if len(parsed) != 1 {
|
||||
cands := buildToolCallCandidates(text)
|
||||
for i, c := range cands {
|
||||
t.Logf("CAND %d: %s", i, c)
|
||||
repaired := RepairLooseJSON(c)
|
||||
t.Logf(" REPAIRED: %s", repaired)
|
||||
}
|
||||
t.Fatalf("expected 1 tool call, got %d. Candidates: %v", len(parsed), buildToolCallCandidates(text))
|
||||
}
|
||||
|
||||
if parsed[0].Name != "todowrite" {
|
||||
t.Errorf("expected tool name 'todowrite', got %q", parsed[0].Name)
|
||||
}
|
||||
|
||||
todos, ok := parsed[0].Input["todos"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected 'todos' to be parsed as a list, got %T: %#v", parsed[0].Input["todos"], parsed[0].Input["todos"])
|
||||
}
|
||||
if len(todos) != 3 {
|
||||
t.Errorf("expected 3 todo items, got %d", len(todos))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsWithMixedWindowsPaths(t *testing.T) {
|
||||
// 更复杂的案例：嵌套 JSON 字符串中的反斜杠未转义
|
||||
text := `关键约束: 1. Windows UTF-8编码处理 2. 必须用绝对路径导入 D:\git_codes\ds2api\misc
|
||||
{tool_calls: [{"name": "write_file", "input": "{\"path\": \"D:\\git_codes\\ds2api\\misc\\queens.py\", \"content\": \"print('hello')\"}"}]}`
|
||||
|
||||
availableTools := []string{"write_file"}
|
||||
parsed := ParseToolCalls(text, availableTools)
|
||||
|
||||
if len(parsed) != 1 {
|
||||
t.Fatalf("expected 1 tool call from mixed text with paths, got %d", len(parsed))
|
||||
}
|
||||
|
||||
path, _ := parsed[0].Input["path"].(string)
|
||||
// 在解析后的 Go map 中，反斜杠应该被还原
|
||||
if !strings.Contains(path, "D:\\git_codes") && !strings.Contains(path, "D:/git_codes") {
|
||||
t.Errorf("expected path to contain Windows style separators, got %q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallInputRepairsControlCharsInPath(t *testing.T) {
|
||||
in := `{"path":"D:\tmp\new\readme.txt","content":"line1\nline2"}`
|
||||
parsed := parseToolCallInput(in)
|
||||
|
||||
path, ok := parsed["path"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("expected path string in parsed input, got %#v", parsed["path"])
|
||||
}
|
||||
if path != `D:\tmp\new\readme.txt` {
|
||||
t.Fatalf("expected repaired windows path, got %q", path)
|
||||
}
|
||||
|
||||
content, ok := parsed["content"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("expected content string in parsed input, got %#v", parsed["content"])
|
||||
}
|
||||
if content != "line1\nline2" {
|
||||
t.Fatalf("expected non-path field to keep decoded escapes, got %q", content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairLooseJSONWithNestedObjects(t *testing.T) {
|
||||
// 测试嵌套对象的修复：DeepSeek 幻觉输出，每个元素内部包含嵌套 {}
|
||||
// 注意：正则只支持单层嵌套，不支持更深层次的嵌套
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
// 1. 单层嵌套对象（核心修复目标）
|
||||
{
|
||||
name: "单层嵌套 - 2个元素",
|
||||
input: `"todos": {"content": "研究算法", "input": {"q": "8 queens"}}, {"content": "实现", "input": {"path": "queens.py"}}`,
|
||||
expected: `"todos": [{"content": "研究算法", "input": {"q": "8 queens"}}, {"content": "实现", "input": {"path": "queens.py"}}]`,
|
||||
},
|
||||
// 2. 3个单层嵌套对象
|
||||
{
|
||||
name: "3个单层嵌套对象",
|
||||
input: `"items": {"a": {"x":1}}, {"b": {"y":2}}, {"c": {"z":3}}`,
|
||||
expected: `"items": [{"a": {"x":1}}, {"b": {"y":2}}, {"c": {"z":3}}]`,
|
||||
},
|
||||
// 3. 混合嵌套：有些字段是对象，有些是原始值
|
||||
{
|
||||
name: "混合嵌套 - 对象和原始值混合",
|
||||
input: `"items": {"name": "test", "config": {"timeout": 30}}, {"name": "test2", "config": {"timeout": 60}}`,
|
||||
expected: `"items": [{"name": "test", "config": {"timeout": 30}}, {"name": "test2", "config": {"timeout": 60}}]`,
|
||||
},
|
||||
// 4. 4个嵌套对象（边界测试）
|
||||
{
|
||||
name: "4个嵌套对象",
|
||||
input: `"todos": {"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}`,
|
||||
expected: `"todos": [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}]`,
|
||||
},
|
||||
// 5. DeepSeek 典型幻觉：无空格逗号分隔
|
||||
{
|
||||
name: "无空格逗号分隔",
|
||||
input: `"results": {"name": "a"}, {"name": "b"}, {"name": "c"}`,
|
||||
expected: `"results": [{"name": "a"}, {"name": "b"}, {"name": "c"}]`,
|
||||
},
|
||||
// 6. 嵌套数组（数组在对象内，不是深层嵌套）
|
||||
{
|
||||
name: "对象内包含数组",
|
||||
input: `"data": {"items": [1,2,3]}, {"items": [4,5,6]}`,
|
||||
expected: `"data": [{"items": [1,2,3]}, {"items": [4,5,6]}]`,
|
||||
},
|
||||
// 7. 真实的 DeepSeek 8皇后问题输出
|
||||
{
|
||||
name: "DeepSeek 8皇后真实输出",
|
||||
input: `"todos": {"content": "研究8皇后算法", "status": "pending"}, {"content": "实现Python脚本", "status": "pending"}, {"content": "验证结果", "status": "pending"}`,
|
||||
expected: `"todos": [{"content": "研究8皇后算法", "status": "pending"}, {"content": "实现Python脚本", "status": "pending"}, {"content": "验证结果", "status": "pending"}]`,
|
||||
},
|
||||
// 8. 简单无嵌套对象（回归测试）
|
||||
{
|
||||
name: "简单无嵌套对象",
|
||||
input: `"items": {"a": 1}, {"b": 2}`,
|
||||
expected: `"items": [{"a": 1}, {"b": 2}]`,
|
||||
},
|
||||
// 9. 更复杂的单层嵌套
|
||||
{
|
||||
name: "复杂单层嵌套",
|
||||
input: `"functions": {"name": "execute", "input": {"command": "ls"}}, {"name": "read", "input": {"file": "a.txt"}}`,
|
||||
expected: `"functions": [{"name": "execute", "input": {"command": "ls"}}, {"name": "read", "input": {"file": "a.txt"}}]`,
|
||||
},
|
||||
// 10. 5个嵌套对象
|
||||
{
|
||||
name: "5个嵌套对象",
|
||||
input: `"tasks": {"id":1}, {"id":2}, {"id":3}, {"id":4}, {"id":5}`,
|
||||
expected: `"tasks": [{"id":1}, {"id":2}, {"id":3}, {"id":4}, {"id":5}]`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := RepairLooseJSON(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("[%s] RepairLooseJSON with nested objects:\n input: %s\n got: %s\n expected: %s", tt.name, tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var textKVNamePattern = regexp.MustCompile(`(?is)function\.name:\s*([a-zA-Z0-9_\-.]+)`)
|
||||
|
||||
func parseTextKVToolCalls(text string) []ParsedToolCall {
|
||||
var out []ParsedToolCall
|
||||
matches := textKVNamePattern.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for i, match := range matches {
|
||||
name := text[match[2]:match[3]]
|
||||
|
||||
offset := match[1]
|
||||
endSearch := len(text)
|
||||
if i+1 < len(matches) {
|
||||
endSearch = matches[i+1][0]
|
||||
}
|
||||
|
||||
searchArea := text[offset:endSearch]
|
||||
argIdx := strings.Index(searchArea, "function.arguments:")
|
||||
if argIdx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
startIdx := offset + argIdx + len("function.arguments:")
|
||||
braceIdx := strings.IndexByte(text[startIdx:endSearch], '{')
|
||||
if braceIdx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
actualStart := startIdx + braceIdx
|
||||
objJson, _, ok := extractJSONObject(text, actualStart)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
input := parseToolCallInput(objJson)
|
||||
out = append(out, ParsedToolCall{
|
||||
Name: name,
|
||||
Input: input,
|
||||
})
|
||||
}
|
||||
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseTextKVToolCalls_Basic(t *testing.T) {
|
||||
text := `
|
||||
status: already_called
|
||||
origin: assistant
|
||||
not_user_input: true
|
||||
tool_call_id: call_3fcd15235eb94f7eae3a8de5a9cfa36b
|
||||
function.name: execute_command
|
||||
function.arguments: {"command":"cd scripts && python check_syntax.py example.py","cwd":null,"timeout":30}
|
||||
|
||||
Some other text thinking...
|
||||
`
|
||||
calls := ParseToolCalls(text, []string{"execute_command"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Name != "execute_command" {
|
||||
t.Fatalf("unexpected name: %s", calls[0].Name)
|
||||
}
|
||||
if calls[0].Input["command"] != "cd scripts && python check_syntax.py example.py" {
|
||||
t.Fatalf("unexpected command arg: %v", calls[0].Input["command"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTextKVToolCalls_Multiple(t *testing.T) {
|
||||
text := `
|
||||
function.name: read_file
|
||||
function.arguments: {
|
||||
"path": "abc.txt"
|
||||
}
|
||||
|
||||
function.name: bash
|
||||
function.arguments: {"command": "ls"}
|
||||
`
|
||||
calls := ParseToolCalls(text, []string{"read_file", "bash"})
|
||||
if len(calls) != 2 {
|
||||
t.Fatalf("expected 2 calls, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Name != "read_file" {
|
||||
t.Fatalf("unexpected 1st name: %s", calls[0].Name)
|
||||
}
|
||||
if calls[1].Name != "bash" {
|
||||
t.Fatalf("unexpected 2nd name: %s", calls[1].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTextKVToolCalls_Standalone(t *testing.T) {
|
||||
text := "function.name: read_file\nfunction.arguments: {\"path\":\"README.md\"}"
|
||||
calls := ParseStandaloneToolCalls(text, []string{"read_file"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Name != "read_file" {
|
||||
t.Fatalf("unexpected name: %s", calls[0].Name)
|
||||
}
|
||||
}
|
||||
@@ -356,89 +356,3 @@ func TestConvertClaudeToDeepSeekOpusUsesSlowMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── FormatOpenAIStreamToolCalls ─────────────────────────────────────
|
||||
|
||||
func TestFormatOpenAIStreamToolCalls(t *testing.T) {
|
||||
formatted := FormatOpenAIStreamToolCalls([]ParsedToolCall{
|
||||
{Name: "search", Input: map[string]any{"q": "test"}},
|
||||
})
|
||||
if len(formatted) != 1 {
|
||||
t.Fatalf("expected 1, got %d", len(formatted))
|
||||
}
|
||||
fn, _ := formatted[0]["function"].(map[string]any)
|
||||
if fn["name"] != "search" {
|
||||
t.Fatalf("unexpected function name: %#v", fn)
|
||||
}
|
||||
if formatted[0]["index"] != 0 {
|
||||
t.Fatalf("expected index 0, got %v", formatted[0]["index"])
|
||||
}
|
||||
}
|
||||
|
||||
// ─── ParseToolCalls more edge cases ──────────────────────────────────
|
||||
|
||||
func TestParseToolCallsNoToolNames(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}}]}`
|
||||
calls := ParseToolCalls(text, nil)
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call with nil tool names, got %d", len(calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsEmptyText(t *testing.T) {
|
||||
calls := ParseToolCalls("", []string{"search"})
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("expected 0 calls for empty text, got %d", len(calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsMultipleTools(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"search","input":{"q":"go"}},{"name":"get_weather","input":{"city":"beijing"}}]}`
|
||||
calls := ParseToolCalls(text, []string{"search", "get_weather"})
|
||||
if len(calls) != 2 {
|
||||
t.Fatalf("expected 2 calls, got %d", len(calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsInputAsString(t *testing.T) {
|
||||
text := `{"tool_calls":[{"name":"search","input":"{\"q\":\"golang\"}"}]}`
|
||||
calls := ParseToolCalls(text, []string{"search"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Input["q"] != "golang" {
|
||||
t.Fatalf("expected parsed string input, got %#v", calls[0].Input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseToolCallsWithFunctionWrapper(t *testing.T) {
|
||||
text := `{"tool_calls":[{"function":{"name":"calc","arguments":{"x":1,"y":2}}}]}`
|
||||
calls := ParseToolCalls(text, []string{"calc"})
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", len(calls))
|
||||
}
|
||||
if calls[0].Name != "calc" {
|
||||
t.Fatalf("expected calc, got %q", calls[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStandaloneToolCallsFencedCodeBlock(t *testing.T) {
|
||||
fenced := "Here's an example:\n```json\n{\"tool_calls\":[{\"name\":\"search\",\"input\":{\"q\":\"go\"}}]}\n```\nDon't execute this."
|
||||
calls := ParseStandaloneToolCalls(fenced, []string{"search"})
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("expected fenced code block to be ignored, got %d calls", len(calls))
|
||||
}
|
||||
}
|
||||
|
||||
// ─── looksLikeToolExampleContext ─────────────────────────────────────
|
||||
|
||||
func TestLooksLikeToolExampleContextNone(t *testing.T) {
|
||||
if looksLikeToolExampleContext("I will call the tool now") {
|
||||
t.Fatal("expected false for non-example context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLooksLikeToolExampleContextFenced(t *testing.T) {
|
||||
if !looksLikeToolExampleContext("```json") {
|
||||
t.Fatal("expected true for fenced code block context")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user