refactor: differentiate reference marker handling between stream and non-stream modes

- Stream: strip both and [reference:N] markers to prevent
  leaking partial link metadata during incremental output
- Non-stream: convert citation/reference markers to Markdown links for
  Claude Messages, Gemini generateContent, and OpenAI Chat/Responses
- Remove StripReferenceMarkers option from call sites; behavior is now
  determined automatically by stream vs non-stream context
- Extend JS runtime stripReferenceMarkersText() to also match [citation:N]
- Add tests for streaming marker stripping and non-stream link conversion

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
CJACK
2026-05-03 17:53:49 +08:00
parent c099a6f7bf
commit 112bedb05d
14 changed files with 99 additions and 52 deletions

View File

@@ -89,11 +89,11 @@ func (a *StreamAccumulator) applyTextPart(text string) StreamPartDelta {
}
a.RawText.WriteString(rawTrimmed)
delta := StreamPartDelta{Type: "text", RawText: rawTrimmed}
cleanedText := CleanVisibleOutput(rawTrimmed, a.StripReferenceMarkers)
if a.SearchEnabled && sse.IsCitation(cleanedText) {
if a.SearchEnabled && sse.IsCitation(rawTrimmed) {
delta.CitationOnly = true
return delta
}
cleanedText := CleanVisibleOutput(rawTrimmed, a.StripReferenceMarkers)
trimmed := sse.TrimContinuationOverlapFromBuilder(&a.Text, cleanedText)
if trimmed == "" {
return delta

View File

@@ -95,3 +95,21 @@ func TestStreamAccumulatorSuppressesCitationTextWhenSearchEnabled(t *testing.T)
t.Fatalf("visible text = %q", got)
}
}
func TestStreamAccumulatorStripsInlineCitationAndReferenceMarkers(t *testing.T) {
acc := StreamAccumulator{SearchEnabled: true, StripReferenceMarkers: true}
result := acc.Apply(sse.LineResult{
Parsed: true,
Parts: []sse.ContentPart{{Type: "text", Text: "广州天气[citation:1] 多云[reference:0]"}},
})
if !result.ContentSeen {
t.Fatalf("expected marker chunk to mark upstream content")
}
if got := acc.Text.String(); got != "广州天气 多云" {
t.Fatalf("visible text = %q", got)
}
if len(result.Parts) != 1 || result.Parts[0].VisibleText != "广州天气 多云" {
t.Fatalf("unexpected parts: %#v", result.Parts)
}
}