refactor: differentiate reference marker handling between stream and non-stream modes

- Stream: strip both and [reference:N] markers to prevent
  leaking partial link metadata during incremental output
- Non-stream: convert citation/reference markers to Markdown links for
  Claude Messages, Gemini generateContent, and OpenAI Chat/Responses
- Remove StripReferenceMarkers option from call sites; behavior is now
  determined automatically by stream vs non-stream context
- Extend JS runtime stripReferenceMarkersText() to also match [citation:N]
- Add tests for streaming marker stripping and non-stream link conversion

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
CJACK
2026-05-03 17:53:49 +08:00
parent c099a6f7bf
commit 112bedb05d
14 changed files with 99 additions and 52 deletions

View File

@@ -2,19 +2,18 @@ package textclean
import "regexp"
var referenceMarkerPattern = regexp.MustCompile(`(?i)\[reference:\s*\d+\]`)
var citationReferenceMarkerPattern = regexp.MustCompile(`(?i)\[(citation|reference):\s*\d+\]`)
func StripReferenceMarkers(text string) string {
if text == "" {
return text
}
return referenceMarkerPattern.ReplaceAllString(text, "")
return citationReferenceMarkerPattern.ReplaceAllString(text, "")
}
// StripReferenceMarkersEnabled returns true while reference-marker
// stripping remains the fixed runtime default. When the behaviour is
// eventually removed this function can be deleted and callers can drop
// the conditional.
// StripReferenceMarkersEnabled returns the default for streaming surfaces,
// where partial citation/reference markers are hidden before the final
// link metadata is available.
func StripReferenceMarkersEnabled() bool {
return true
}