refactor: replace processed output comparison with baseline-based validation in SSE simulator

This commit is contained in:
CJACK
2026-04-05 01:34:06 +08:00
parent 93879c9808
commit 0bebb4b28d
20 changed files with 3014 additions and 3853 deletions

View File

@@ -8,28 +8,32 @@ func filterLeakedContentFilterParts(parts []ContentPart) []ContentPart {
}
out := make([]ContentPart, 0, len(parts))
for _, p := range parts {
cleaned := stripLeakedContentFilterSuffix(p.Text)
if shouldDropCleanedLeakedChunk(cleaned) {
cleaned, stripped := stripLeakedContentFilterSuffix(p.Text)
// Only drop the chunk when we actually stripped a leaked CONTENT_FILTER
// suffix. Plain whitespace chunks are valid SSE content and must stay.
if stripped && shouldDropCleanedLeakedChunk(cleaned) {
continue
}
p.Text = cleaned
if stripped {
p.Text = cleaned
}
out = append(out, p)
}
return out
}
func stripLeakedContentFilterSuffix(text string) string {
func stripLeakedContentFilterSuffix(text string) (string, bool) {
if text == "" {
return text
return text, false
}
upperText := strings.ToUpper(text)
idx := strings.Index(upperText, "CONTENT_FILTER")
if idx < 0 {
return text
return text, false
}
// Keep "\n" so we don't collapse line structure when the upstream model
// appends leaked CONTENT_FILTER markers after a line break.
return strings.TrimRight(text[:idx], " \t\r")
return strings.TrimRight(text[:idx], " \t\r"), true
}
func shouldDropCleanedLeakedChunk(cleaned string) bool {

View File

@@ -63,6 +63,16 @@ func TestParseDeepSeekContentLineContent(t *testing.T) {
}
}
func TestParseDeepSeekContentLinePreservesSpaceOnlyChunk(t *testing.T) {
res := ParseDeepSeekContentLine([]byte(`data: {"v":" "}`), false, "text")
if !res.Parsed || res.Stop {
t.Fatalf("expected parsed non-stop result: %#v", res)
}
if len(res.Parts) != 1 || res.Parts[0].Text != " " || res.Parts[0].Type != "text" {
t.Fatalf("unexpected parts for space-only chunk: %#v", res.Parts)
}
}
func TestParseDeepSeekContentLineStripsLeakedContentFilterSuffix(t *testing.T) {
res := ParseDeepSeekContentLine([]byte(`data: {"p":"response/content","v":"正常输出CONTENT_FILTER你好这个问题我暂时无法回答"}`), false, "text")
if !res.Parsed || res.Stop {