lisone/ds2api
1
0
Fork 0
mirror of https://github.com/CJackHwang/ds2api.git synced 2026-05-05 00:45:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add sanitization for think tags and BOS markers in leaked output and update golang.org/x/net dependency

Browse Source
This commit is contained in:
CJACK
2026-04-12 17:43:57 +08:00
parent d759804c33
commit 0a23c77ff7
3 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -18,7 +18,7 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
golang.org/x/crypto v0.49.0 // indirect
golang.org/x/net v0.52.0 // indirect
golang.org/x/net v0.52.0
golang.org/x/sys v0.42.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

11
internal/adapter/openai/leaked_output_sanitize.go
View File

@@ -8,7 +8,14 @@ var emptyJSONFencePattern = regexp.MustCompile("(?is)```json\\s*```")
var leakedToolCallArrayPattern = regexp.MustCompile(`(?is)\[\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"call[^"]*"\s*,\s*"type"\s*:\s*"function"\s*}\]`)
var leakedToolResultBlobPattern = regexp.MustCompile(`(?is)<\s*\|\s*tool\s*\|\s*>\s*\{[\s\S]*?"tool_call_id"\s*:\s*"call[^"]*"\s*}`)
// leakedMetaMarkerPattern matches DeepSeek special tokens in BOTH forms:
var leakedThinkTagPattern = regexp.MustCompile(`(?i)</?think>`)
// leakedBOSMarkerPattern matches DeepSeek BOS markers in BOTH forms:
// - ASCII underscore: <｜begin_of_sentence｜>
// - U+2581 variant: <｜begin▁of▁sentence｜>
var leakedBOSMarkerPattern = regexp.MustCompile(`(?i)<[｜\|]\s*begin[_▁]of[_▁]sentence\s*[｜\|]>`)
// leakedMetaMarkerPattern matches the remaining DeepSeek special tokens in BOTH forms:
// - ASCII underscore: <｜end_of_sentence｜>, <｜end_of_toolresults｜>, <｜end_of_instructions｜>
// - U+2581 variant: <｜end▁of▁sentence｜>, <｜end▁of▁toolresults｜>, <｜end▁of▁instructions｜>
var leakedMetaMarkerPattern = regexp.MustCompile(`(?i)<[｜\|]\s*(?:assistant|tool|end[_▁]of[_▁]sentence|end[_▁]of[_▁]thinking|end[_▁]of[_▁]toolresults|end[_▁]of[_▁]instructions)\s*[｜\|]>`)
@@ -35,6 +42,8 @@ func sanitizeLeakedOutput(text string) string {
out := emptyJSONFencePattern.ReplaceAllString(text, "")
out = leakedToolCallArrayPattern.ReplaceAllString(out, "")
out = leakedToolResultBlobPattern.ReplaceAllString(out, "")
out = leakedThinkTagPattern.ReplaceAllString(out, "")
out = leakedBOSMarkerPattern.ReplaceAllString(out, "")
out = leakedMetaMarkerPattern.ReplaceAllString(out, "")
out = sanitizeLeakedAgentXMLBlocks(out)
return out

8
internal/adapter/openai/leaked_output_sanitize_test.go
View File

@@ -26,6 +26,14 @@ func TestSanitizeLeakedOutputRemovesStandaloneMetaMarkers(t *testing.T) {
}
}
func TestSanitizeLeakedOutputRemovesThinkAndBosMarkers(t *testing.T) {
raw := "A<think>B</think>C<｜begin▁of▁sentence｜>D<| begin_of_sentence |>E<｜begin_of_sentence｜>F"
got := sanitizeLeakedOutput(raw)
if got != "ABCDEF" {
t.Fatalf("unexpected sanitize result for think/BOS markers: %q", got)
}
}
func TestSanitizeLeakedOutputRemovesAgentXMLLeaks(t *testing.T) {
raw := "Done.<attempt_completion><result>Some final answer</result></attempt_completion>"
got := sanitizeLeakedOutput(raw)