refactor: replace history transcript format with numbered sections and rename upload file to HISTORY.txt

This commit is contained in:
CJACK
2026-05-01 21:15:17 +08:00
parent 0a6ef8e3f2
commit df1cfac9bc
7 changed files with 164 additions and 45 deletions

View File

@@ -249,7 +249,7 @@ OpenAI 文件相关实现:
兼容层现在只保留 `current_input_file` 这一种拆分方式;旧的 `history_split` 已废弃,只保留为兼容旧配置的字段,不再参与请求处理。
- `current_input_file` 默认开启;它用于把“完整上下文”合并进 `history.txt` 上下文文件。当最新 user turn 的纯文本长度达到 `current_input_file.min_chars`(默认 `0`)时,兼容层会上传一个文件名为 `history.txt` 的上下文文件,并在 live prompt 中只保留一个中性的 user 消息要求模型直接回答最新请求,不再暴露文件名或要求模型读取本地文件。
- `current_input_file` 默认开启;它用于把“完整上下文”合并进 `HISTORY.txt` 上下文文件。当最新 user turn 的纯文本长度达到 `current_input_file.min_chars`(默认 `0`)时,兼容层会上传一个文件名为 `HISTORY.txt` 的上下文文件。文件内容会先做 OpenAI 消息标准化,再序列化成按轮次编号的 `HISTORY.txt` 风格 transcript带有 `# HISTORY.txt` 标题和 `=== N. ROLE ===` 分段;live prompt 中只保留一个中性的 user 消息要求模型直接回答最新请求,不再暴露文件名或要求模型读取本地文件。
- 如果 `current_input_file.enabled=false`,请求会直接透传,不上传任何拆分上下文文件。
- 旧的 `history_split.enabled` / `history_split.trigger_after_turns` 会被读取进配置对象以保持兼容,但不会触发拆分上传,也不会影响 `current_input_file` 的默认开启。
- 即使触发 `current_input_file` 后 live prompt 被缩短,对客户端回包里的上下文 token 统计,仍会沿用**拆分前的完整 prompt 语义**做计数,而不是按缩短后的占位 prompt 计算;否则会把真实上下文显著算小。
@@ -263,11 +263,24 @@ OpenAI 文件相关实现:
- 旧历史拆分兼容壳:
[internal/httpapi/openai/history/history_split.go](../internal/httpapi/openai/history/history_split.go)
当前输入转文件启用并触发时,上传文件的真实文件名是 `history.txt`,文件内容是完整 `messages` 上下文;它仍会先用 OpenAI 消息标准化和 DeepSeek 角色标记序列化,并直接作为 `history.txt` 的纯文本内容上传(不再注入文件边界标签):
当前输入转文件启用并触发时,上传文件的真实文件名是 `HISTORY.txt`,文件内容是完整 `messages` 上下文;它仍会先用 OpenAI 消息标准化和 DeepSeek 角色标记序列化,再按轮次编号成 `HISTORY.txt` 风格的 transcript(不再注入文件边界标签):
```text
[uploaded filename]: history.txt
<begin▁of▁sentence><System>...<User>...<Assistant>...<Tool>...<User>...
[uploaded filename]: HISTORY.txt
# HISTORY.txt
Prior conversation history and tool progress.
=== 1. SYSTEM ===
...
=== 2. USER ===
...
=== 3. ASSISTANT ===
...
=== 4. TOOL ===
...
```
开启后,请求的 live prompt 不再直接内联完整上下文,而是保留一个 user role 的短提示,提示模型基于已提供上下文直接回答最新请求;上传后的 `file_id` 会进入 `ref_file_ids`
@@ -334,7 +347,7 @@ OpenAI 文件相关实现:
- 大部分结构化语义被压进 `prompt`
- 文件保持文件
- 需要时把完整上下文拆进 `history.txt` 上下文文件
- 需要时把完整上下文拆进 `HISTORY.txt` 上下文文件,并按轮次编号成 transcript
## 12. 修改时必须同步本文档的场景
@@ -347,7 +360,7 @@ OpenAI 文件相关实现:
- tool result 注入方式变更
- tool prompt 模板或 tool_choice 约束变更
- inline 文件上传 / 文件引用收集规则变更
- current input file 触发条件、上传格式、`history.txt` 包装格式变更
- current input file 触发条件、上传格式、`HISTORY.txt` transcript 结构变更
-`history_split` 兼容逻辑的读取、忽略或退化行为变更
- completion payload 字段语义变更
- Claude / Gemini 对这套统一语义的复用关系变更

View File

@@ -311,8 +311,8 @@ func TestChatCompletionsCurrentInputFilePersistsNeutralPrompt(t *testing.T) {
if len(ds.uploadCalls) != 1 {
t.Fatalf("expected current input upload to happen, got %d", len(ds.uploadCalls))
}
if ds.uploadCalls[0].Filename != "history.txt" {
t.Fatalf("expected history.txt upload, got %q", ds.uploadCalls[0].Filename)
if ds.uploadCalls[0].Filename != "HISTORY.txt" {
t.Fatalf("expected HISTORY.txt upload, got %q", ds.uploadCalls[0].Filename)
}
if full.HistoryText != string(ds.uploadCalls[0].Data) {
t.Fatalf("expected uploaded current input file to be persisted in history text")

View File

@@ -62,7 +62,7 @@ func (s Service) ApplyCurrentInputFile(ctx context.Context, a *auth.RequestAuth,
stdReq.RefFileIDs = prependUniqueRefFileID(stdReq.RefFileIDs, fileID)
stdReq.FinalPrompt, stdReq.ToolNames = promptcompat.BuildOpenAIPrompt(messages, stdReq.ToolsRaw, "", stdReq.ToolChoice, stdReq.Thinking)
// Token accounting must reflect the actual downstream context:
// the uploaded history.txt file content + the neutral live prompt.
// the uploaded HISTORY.txt file content + the neutral live prompt.
stdReq.PromptTokenText = fileText + "\n" + stdReq.FinalPrompt
return stdReq, nil
}

View File

@@ -61,26 +61,33 @@ func (streamStatusManagedAuthStub) DetermineCaller(_ *http.Request) (*auth.Reque
func (streamStatusManagedAuthStub) Release(_ *auth.RequestAuth) {}
func TestBuildOpenAICurrentInputContextTranscriptUsesInjectedFileWrapper(t *testing.T) {
func TestBuildOpenAICurrentInputContextTranscriptUsesNumberedHistorySections(t *testing.T) {
_, historyMessages := splitOpenAIHistoryMessages(historySplitTestMessages(), 1)
transcript := buildOpenAICurrentInputContextTranscript(historyMessages)
if strings.Contains(transcript, "[file content end]") || strings.Contains(transcript, "[file content begin]") || strings.Contains(transcript, "[file name]:") {
t.Fatalf("expected plain transcript without file wrapper tags, got %q", transcript)
t.Fatalf("expected transcript without file wrapper tags, got %q", transcript)
}
if !strings.Contains(transcript, "<begin▁of▁sentence>") {
t.Fatalf("expected serialized conversation markers, got %q", transcript)
if !strings.Contains(transcript, "# HISTORY.txt") {
t.Fatalf("expected history transcript header, got %q", transcript)
}
if !strings.Contains(transcript, "first user turn") || !strings.Contains(transcript, "tool result") {
t.Fatalf("expected historical turns preserved, got %q", transcript)
if !strings.Contains(transcript, "Prior conversation history and tool progress.") {
t.Fatalf("expected history transcript description, got %q", transcript)
}
if !strings.Contains(transcript, "[reasoning_content]") || !strings.Contains(transcript, "hidden reasoning") {
t.Fatalf("expected reasoning block preserved, got %q", transcript)
for _, want := range []string{
"=== 1. USER ===",
"=== 2. ASSISTANT ===",
"=== 3. TOOL ===",
"first user turn",
"tool result",
"[reasoning_content]",
"hidden reasoning",
"<|DSML|tool_calls>",
} {
if !strings.Contains(transcript, want) {
t.Fatalf("expected transcript to contain %q, got %q", want, transcript)
}
}
if !strings.Contains(transcript, "<|DSML|tool_calls>") {
t.Fatalf("expected tool calls preserved, got %q", transcript)
}
}
func TestSplitOpenAIHistoryMessagesUsesLatestUserTurn(t *testing.T) {
@@ -243,7 +250,7 @@ func TestApplyCurrentInputFileDisabledPassThrough(t *testing.T) {
}
}
func TestApplyCurrentInputFileUploadsFirstTurnWithInjectedWrapper(t *testing.T) {
func TestApplyCurrentInputFileUploadsFirstTurnWithNumberedHistoryTranscript(t *testing.T) {
ds := &inlineUploadDSStub{}
h := &openAITestSurface{
Store: mockOpenAIConfig{
@@ -273,15 +280,21 @@ func TestApplyCurrentInputFileUploadsFirstTurnWithInjectedWrapper(t *testing.T)
t.Fatalf("expected 1 current input upload, got %d", len(ds.uploadCalls))
}
upload := ds.uploadCalls[0]
if upload.Filename != "history.txt" {
if upload.Filename != "HISTORY.txt" {
t.Fatalf("unexpected upload filename: %q", upload.Filename)
}
uploadedText := string(upload.Data)
if strings.Contains(uploadedText, "[file content end]") || strings.Contains(uploadedText, "[file content begin]") || strings.Contains(uploadedText, "[file name]:") {
t.Fatalf("expected uploaded transcript without file wrapper tags, got %q", uploadedText)
}
if !strings.Contains(uploadedText, "<begin▁of▁sentence><User>first turn content that is long enough") {
t.Fatalf("expected serialized current user turn markers, got %q", uploadedText)
for _, want := range []string{
"# HISTORY.txt",
"=== 1. USER ===",
"first turn content that is long enough",
} {
if !strings.Contains(uploadedText, want) {
t.Fatalf("expected uploaded transcript to contain %q, got %q", want, uploadedText)
}
}
if !strings.Contains(uploadedText, promptcompat.ThinkingInjectionMarker) {
t.Fatalf("expected thinking injection in current input file, got %q", uploadedText)
@@ -290,7 +303,7 @@ func TestApplyCurrentInputFileUploadsFirstTurnWithInjectedWrapper(t *testing.T)
if strings.Contains(out.FinalPrompt, "first turn content that is long enough") {
t.Fatalf("expected current input text to be replaced in live prompt, got %s", out.FinalPrompt)
}
if strings.Contains(out.FinalPrompt, "CURRENT_USER_INPUT.txt") || strings.Contains(out.FinalPrompt, "history.txt") || strings.Contains(out.FinalPrompt, "Read that file") {
if strings.Contains(out.FinalPrompt, "CURRENT_USER_INPUT.txt") || strings.Contains(out.FinalPrompt, "history.txt") || strings.Contains(out.FinalPrompt, "HISTORY.txt") || strings.Contains(out.FinalPrompt, "Read that file") {
t.Fatalf("expected live prompt not to instruct file reads, got %s", out.FinalPrompt)
}
if !strings.Contains(out.FinalPrompt, "Answer the latest user request directly.") {
@@ -302,6 +315,9 @@ func TestApplyCurrentInputFileUploadsFirstTurnWithInjectedWrapper(t *testing.T)
if !strings.Contains(out.PromptTokenText, "first turn content that is long enough") {
t.Fatalf("expected prompt token text to preserve original full context, got %q", out.PromptTokenText)
}
if !strings.Contains(out.PromptTokenText, "# HISTORY.txt") || !strings.Contains(out.PromptTokenText, "=== 1. USER ===") {
t.Fatalf("expected prompt token text to include numbered history transcript, got %q", out.PromptTokenText)
}
}
func TestApplyCurrentInputFilePreservesFullContextPromptForTokenCounting(t *testing.T) {
@@ -337,7 +353,10 @@ func TestApplyCurrentInputFilePreservesFullContextPromptForTokenCounting(t *test
t.Fatalf("expected prompt token text to contain file context with full conversation, got %q", out.PromptTokenText)
}
if strings.Contains(out.PromptTokenText, "[file content end]") || strings.Contains(out.PromptTokenText, "[file name]:") {
t.Fatalf("expected prompt token text to use raw transcript without wrapper tags, got %q", out.PromptTokenText)
t.Fatalf("expected prompt token text to omit file wrapper tags, got %q", out.PromptTokenText)
}
if !strings.Contains(out.PromptTokenText, "# HISTORY.txt") || !strings.Contains(out.PromptTokenText, "=== 1. SYSTEM ===") {
t.Fatalf("expected prompt token text to include numbered history transcript, got %q", out.PromptTokenText)
}
if !strings.Contains(out.PromptTokenText, "Answer the latest user request directly.") {
t.Fatalf("expected prompt token text to also include neutral live prompt, got %q", out.PromptTokenText)
@@ -378,16 +397,16 @@ func TestApplyCurrentInputFileUploadsFullContextFile(t *testing.T) {
t.Fatalf("expected one current input upload, got %d", len(ds.uploadCalls))
}
upload := ds.uploadCalls[0]
if upload.Filename != "history.txt" {
t.Fatalf("expected history.txt upload, got %q", upload.Filename)
if upload.Filename != "HISTORY.txt" {
t.Fatalf("expected HISTORY.txt upload, got %q", upload.Filename)
}
uploadedText := string(upload.Data)
for _, want := range []string{"system instructions", "first user turn", "hidden reasoning", "tool result", "latest user turn", promptcompat.ThinkingInjectionMarker} {
for _, want := range []string{"# HISTORY.txt", "=== 1. SYSTEM ===", "=== 2. USER ===", "=== 3. ASSISTANT ===", "=== 4. TOOL ===", "=== 5. USER ===", "system instructions", "first user turn", "hidden reasoning", "tool result", "latest user turn", promptcompat.ThinkingInjectionMarker} {
if !strings.Contains(uploadedText, want) {
t.Fatalf("expected full context file to contain %q, got %q", want, uploadedText)
}
}
if strings.Contains(out.FinalPrompt, "first user turn") || strings.Contains(out.FinalPrompt, "latest user turn") || strings.Contains(out.FinalPrompt, "CURRENT_USER_INPUT.txt") || strings.Contains(out.FinalPrompt, "history.txt") || strings.Contains(out.FinalPrompt, "Read that file") {
if strings.Contains(out.FinalPrompt, "first user turn") || strings.Contains(out.FinalPrompt, "latest user turn") || strings.Contains(out.FinalPrompt, "CURRENT_USER_INPUT.txt") || strings.Contains(out.FinalPrompt, "history.txt") || strings.Contains(out.FinalPrompt, "HISTORY.txt") || strings.Contains(out.FinalPrompt, "Read that file") {
t.Fatalf("expected live prompt to use only a neutral continuation instruction, got %s", out.FinalPrompt)
}
if !strings.Contains(out.FinalPrompt, "Answer the latest user request directly.") {
@@ -423,6 +442,9 @@ func TestApplyCurrentInputFileCarriesHistoryText(t *testing.T) {
if out.HistoryText != string(ds.uploadCalls[0].Data) {
t.Fatalf("expected current input file flow to preserve uploaded text in history, got %q", out.HistoryText)
}
if !strings.Contains(out.HistoryText, "# HISTORY.txt") || !strings.Contains(out.HistoryText, "=== 1. SYSTEM ===") {
t.Fatalf("expected history text to use numbered transcript format, got %q", out.HistoryText)
}
}
func TestChatCompletionsCurrentInputFileUploadsContextAndKeepsNeutralPrompt(t *testing.T) {
@@ -454,7 +476,7 @@ func TestChatCompletionsCurrentInputFileUploadsContextAndKeepsNeutralPrompt(t *t
t.Fatalf("expected 1 upload call, got %d", len(ds.uploadCalls))
}
upload := ds.uploadCalls[0]
if upload.Filename != "history.txt" {
if upload.Filename != "HISTORY.txt" {
t.Fatalf("unexpected upload filename: %q", upload.Filename)
}
if upload.Purpose != "assistants" {
@@ -462,7 +484,10 @@ func TestChatCompletionsCurrentInputFileUploadsContextAndKeepsNeutralPrompt(t *t
}
historyText := string(upload.Data)
if strings.Contains(historyText, "[file content end]") || strings.Contains(historyText, "[file content begin]") || strings.Contains(historyText, "[file name]:") {
t.Fatalf("expected plain history transcript without wrapper tags, got %s", historyText)
t.Fatalf("expected history transcript without file wrapper tags, got %s", historyText)
}
if !strings.Contains(historyText, "# HISTORY.txt") || !strings.Contains(historyText, "=== 1. SYSTEM ===") {
t.Fatalf("expected history transcript to use numbered sections, got %s", historyText)
}
if !strings.Contains(historyText, "latest user turn") {
t.Fatalf("expected full context to include latest turn, got %s", historyText)
@@ -523,6 +548,10 @@ func TestResponsesCurrentInputFileUploadsContextAndKeepsNeutralPrompt(t *testing
if len(ds.uploadCalls) != 1 {
t.Fatalf("expected 1 upload call, got %d", len(ds.uploadCalls))
}
historyText := string(ds.uploadCalls[0].Data)
if !strings.Contains(historyText, "# HISTORY.txt") || !strings.Contains(historyText, "=== 1. SYSTEM ===") {
t.Fatalf("expected uploaded history text to use numbered transcript format, got %s", historyText)
}
if ds.completionReq == nil {
t.Fatal("expected completion payload to be captured")
}
@@ -669,6 +698,10 @@ func TestCurrentInputFileWorksAcrossAutoDeleteModes(t *testing.T) {
if len(ds.uploadCalls) != 1 {
t.Fatalf("expected current input upload for mode=%s, got %d", mode, len(ds.uploadCalls))
}
historyText := string(ds.uploadCalls[0].Data)
if !strings.Contains(historyText, "# HISTORY.txt") || !strings.Contains(historyText, "=== 1. SYSTEM ===") {
t.Fatalf("expected uploaded history text to use numbered transcript format, got %s", historyText)
}
if ds.completionReq == nil {
t.Fatalf("expected completion payload for mode=%s", mode)
}

View File

@@ -1,35 +1,108 @@
package promptcompat
import (
"fmt"
"strings"
"ds2api/internal/prompt"
)
const CurrentInputContextFilename = "history.txt"
const CurrentInputContextFilename = "HISTORY.txt"
const historyTranscriptTitle = "# HISTORY.txt"
const historyTranscriptSummary = "Prior conversation history and tool progress."
func BuildOpenAIHistoryTranscript(messages []any) string {
return buildOpenAIInjectedFileTranscript(messages)
return buildOpenAIHistoryTranscript(messages)
}
func BuildOpenAICurrentUserInputTranscript(text string) string {
if strings.TrimSpace(text) == "" {
return ""
}
return BuildOpenAICurrentInputContextTranscript([]any{
return buildOpenAIHistoryTranscript([]any{
map[string]any{"role": "user", "content": text},
})
}
func BuildOpenAICurrentInputContextTranscript(messages []any) string {
return buildOpenAIInjectedFileTranscript(messages)
return buildOpenAIHistoryTranscript(messages)
}
func buildOpenAIInjectedFileTranscript(messages []any) string {
normalized := NormalizeOpenAIMessagesForPrompt(messages, "")
transcript := strings.TrimSpace(prompt.MessagesPrepare(normalized))
func buildOpenAIHistoryTranscript(messages []any) string {
if len(messages) == 0 {
return ""
}
var b strings.Builder
b.WriteString(historyTranscriptTitle)
b.WriteString("\n")
b.WriteString(historyTranscriptSummary)
b.WriteString("\n\n")
entry := 0
for _, raw := range messages {
msg, ok := raw.(map[string]any)
if !ok {
continue
}
role := normalizeOpenAIRoleForPrompt(strings.ToLower(strings.TrimSpace(asString(msg["role"]))))
content := strings.TrimSpace(buildOpenAIHistoryEntry(role, msg))
if content == "" {
continue
}
entry++
fmt.Fprintf(&b, "=== %d. %s ===\n%s\n\n", entry, strings.ToUpper(roleLabelForHistory(role)), content)
}
transcript := strings.TrimSpace(b.String())
if transcript == "" {
return ""
}
return transcript
return transcript + "\n"
}
func buildOpenAIHistoryEntry(role string, msg map[string]any) string {
switch role {
case "assistant":
return strings.TrimSpace(buildAssistantContentForPrompt(msg))
case "tool", "function":
return strings.TrimSpace(buildToolHistoryContent(msg))
case "system", "user":
return strings.TrimSpace(NormalizeOpenAIContentForPrompt(msg["content"]))
default:
return strings.TrimSpace(NormalizeOpenAIContentForPrompt(msg["content"]))
}
}
func buildToolHistoryContent(msg map[string]any) string {
content := strings.TrimSpace(NormalizeOpenAIContentForPrompt(msg["content"]))
parts := make([]string, 0, 2)
if name := strings.TrimSpace(asString(msg["name"])); name != "" {
parts = append(parts, "name="+name)
}
if callID := strings.TrimSpace(asString(msg["tool_call_id"])); callID != "" {
parts = append(parts, "tool_call_id="+callID)
}
header := ""
if len(parts) > 0 {
header = "[" + strings.Join(parts, " ") + "]"
}
switch {
case header != "" && content != "":
return header + "\n" + content
case header != "":
return header
default:
return content
}
}
func roleLabelForHistory(role string) string {
role = strings.ToLower(strings.TrimSpace(role))
switch role {
case "function":
return "tool"
case "":
return "unknown"
default:
return role
}
}

View File

@@ -394,7 +394,7 @@
"thinkingInjectionPromptHelp": "Leave empty to use the built-in default prompt shown as the input placeholder.",
"currentInputFileTitle": "Independent Split",
"currentInputFileEnabled": "Independent split (by size)",
"currentInputFileDesc": "Enabled by default. Once the character threshold is reached, upload the full context as a history.txt context file.",
"currentInputFileDesc": "Enabled by default. Once the character threshold is reached, upload the full context as a HISTORY.txt context file.",
"currentInputFileMinChars": "Current input threshold (characters)",
"currentInputFileHelp": "Default is 0, which uses independent split for any non-empty input.",
"compatibilityTitle": "Compatibility",

View File

@@ -394,7 +394,7 @@
"thinkingInjectionPromptHelp": "留空时使用内置默认提示词;默认内容会显示在输入框占位文本中。",
"currentInputFileTitle": "独立拆分",
"currentInputFileEnabled": "独立拆分(按量)",
"currentInputFileDesc": "默认开启。达到字符阈值后,将完整上下文上传为 history.txt 上下文文件。",
"currentInputFileDesc": "默认开启。达到字符阈值后,将完整上下文上传为 HISTORY.txt 上下文文件。",
"currentInputFileMinChars": "当前输入阈值(字符数)",
"currentInputFileHelp": "默认 0表示只要有输入就会使用独立拆分。",
"compatibilityTitle": "兼容性设置",