mirror of
https://github.com/CJackHwang/ds2api.git
synced 2026-05-18 23:25:10 +08:00
refactor: replace processed output comparison with baseline-based validation in SSE simulator
This commit is contained in:
@@ -8,28 +8,32 @@ func filterLeakedContentFilterParts(parts []ContentPart) []ContentPart {
|
||||
}
|
||||
out := make([]ContentPart, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
cleaned := stripLeakedContentFilterSuffix(p.Text)
|
||||
if shouldDropCleanedLeakedChunk(cleaned) {
|
||||
cleaned, stripped := stripLeakedContentFilterSuffix(p.Text)
|
||||
// Only drop the chunk when we actually stripped a leaked CONTENT_FILTER
|
||||
// suffix. Plain whitespace chunks are valid SSE content and must stay.
|
||||
if stripped && shouldDropCleanedLeakedChunk(cleaned) {
|
||||
continue
|
||||
}
|
||||
p.Text = cleaned
|
||||
if stripped {
|
||||
p.Text = cleaned
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stripLeakedContentFilterSuffix(text string) string {
|
||||
func stripLeakedContentFilterSuffix(text string) (string, bool) {
|
||||
if text == "" {
|
||||
return text
|
||||
return text, false
|
||||
}
|
||||
upperText := strings.ToUpper(text)
|
||||
idx := strings.Index(upperText, "CONTENT_FILTER")
|
||||
if idx < 0 {
|
||||
return text
|
||||
return text, false
|
||||
}
|
||||
// Keep "\n" so we don't collapse line structure when the upstream model
|
||||
// appends leaked CONTENT_FILTER markers after a line break.
|
||||
return strings.TrimRight(text[:idx], " \t\r")
|
||||
return strings.TrimRight(text[:idx], " \t\r"), true
|
||||
}
|
||||
|
||||
func shouldDropCleanedLeakedChunk(cleaned string) bool {
|
||||
|
||||
@@ -63,6 +63,16 @@ func TestParseDeepSeekContentLineContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDeepSeekContentLinePreservesSpaceOnlyChunk(t *testing.T) {
|
||||
res := ParseDeepSeekContentLine([]byte(`data: {"v":" "}`), false, "text")
|
||||
if !res.Parsed || res.Stop {
|
||||
t.Fatalf("expected parsed non-stop result: %#v", res)
|
||||
}
|
||||
if len(res.Parts) != 1 || res.Parts[0].Text != " " || res.Parts[0].Type != "text" {
|
||||
t.Fatalf("unexpected parts for space-only chunk: %#v", res.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDeepSeekContentLineStripsLeakedContentFilterSuffix(t *testing.T) {
|
||||
res := ParseDeepSeekContentLine([]byte(`data: {"p":"response/content","v":"正常输出CONTENT_FILTER你好,这个问题我暂时无法回答"}`), false, "text")
|
||||
if !res.Parsed || res.Stop {
|
||||
|
||||
Reference in New Issue
Block a user