From 6959aa29827e6dd43cf70cbb444ab75e3bb6dbaa Mon Sep 17 00:00:00 2001 From: CJACK Date: Mon, 27 Apr 2026 14:37:23 +0800 Subject: [PATCH] feat: add ETag cache optimization, code-split WebUI, and refactor XML tool scanner - Chat history: early 304 via Revision()/DetailRevision() to avoid full snapshot reads - WebUI: lazy-load tab containers with Suspense fallback - Toolstream: split tool_sieve_xml.go into tags.go and scan.go - CI: trigger on main branch, guard cross-build to dev/main pushes only - Docs: add DEVELOPER.md developer quick reference Co-Authored-By: Claude Opus 4.7 --- .github/workflows/quality-gates.yml | 2 + docs/DEVELOPMENT.md | 112 +++++++++++ docs/README.md | 6 +- internal/chathistory/store.go | 28 +++ .../admin/history/handler_chat_history.go | 41 +++- .../history/handler_chat_history_test.go | 9 + internal/toolstream/tool_sieve_xml.go | 176 ------------------ internal/toolstream/tool_sieve_xml_scan.go | 138 ++++++++++++++ internal/toolstream/tool_sieve_xml_tags.go | 43 +++++ webui/src/layout/DashboardShell.jsx | 40 ++-- 10 files changed, 403 insertions(+), 192 deletions(-) create mode 100644 docs/DEVELOPMENT.md create mode 100644 internal/toolstream/tool_sieve_xml_scan.go create mode 100644 internal/toolstream/tool_sieve_xml_tags.go diff --git a/.github/workflows/quality-gates.yml b/.github/workflows/quality-gates.yml index 64e70d5..6d6a9d5 100644 --- a/.github/workflows/quality-gates.yml +++ b/.github/workflows/quality-gates.yml @@ -5,6 +5,7 @@ on: push: branches: - dev + - main permissions: contents: read @@ -114,6 +115,7 @@ jobs: cross-build: name: Release Target Cross-Build + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 0000000..4002e13 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,112 @@ +# DS2API 开发者速查 + +语言 / Language: 中文 + +本文面向维护者和贡献者，用于快速判断“从哪里看、改哪里、跑什么”。架构细节仍以 [ARCHITECTURE.md](./ARCHITECTURE.md) 为准，接口行为以 [API.md](../API.md) 为准。 + +## 1. 本地入口 + +常用启动与检查： + +```bash +# 后端 +go run ./cmd/ds2api + +# WebUI 开发服务器 +npm run dev --prefix webui + +# WebUI 生产构建 +npm run build --prefix webui +``` + +PR 前固定门禁： + +```bash +./scripts/lint.sh +./tests/scripts/check-refactor-line-gate.sh +./tests/scripts/run-unit-all.sh +npm run build --prefix webui +``` + +修改 Go 文件后先运行： + +```bash +gofmt -w +``` + +## 2. 代码定位 + +优先从这些入口顺着调用链看： + +| 目标 | 入口 | +| --- | --- | +| 总路由、CORS、健康检查 | `internal/server/router.go` | +| OpenAI Chat / Responses | `internal/httpapi/openai/chat`、`internal/httpapi/openai/responses` | +| Claude / Gemini 兼容入口 | `internal/httpapi/claude`、`internal/httpapi/gemini` | +| API 请求归一到网页纯文本上下文 | `internal/promptcompat`、`docs/prompt-compatibility.md` | +| 工具调用解析与流式防泄漏 | `internal/toolcall`、`internal/toolstream`、`docs/toolcall-semantics.md` | +| DeepSeek 上游调用、登录、PoW、代理 | `internal/deepseek/client`、`internal/deepseek/transport` | +| 账号池、并发槽位、等待队列 | `internal/account` | +| Admin API | `internal/httpapi/admin` | +| WebUI 页面 | `webui/src/layout/DashboardShell.jsx`、`webui/src/features/*` | +| 服务器端对话记录 | `internal/chathistory`、`internal/httpapi/admin/history` | + +## 3. 常见改动建议 + +- 改接口行为时，同时检查 `API.md` / `API.en.md` 是否需要同步。 +- 改 prompt 兼容链路时，必须同步 `docs/prompt-compatibility.md`。 +- 改 tool call 语义时，同时检查 Go、Node sieve 和 `docs/toolcall-semantics.md`。 +- 改 WebUI 配置项时，同时检查 `webui/src/features/settings`、语言包和 `config.example.json`。 +- 拆分大文件时，保持对外函数签名稳定，并跑 `./tests/scripts/check-refactor-line-gate.sh`。 + +## 4. 故障定位 + +接口请求先看路由入口，再看协议适配层，最后看共享 runtime： + +1. 路由是否命中：`internal/server/router.go` 和对应 `RegisterRoutes`。 +2. 鉴权与账号选择：`internal/auth`、`internal/account`。 +3. 请求归一化：`internal/promptcompat` 或协议转换包。 +4. 上游请求：`internal/deepseek/client`。 +5. 流式输出：`internal/stream`、`internal/sse`、`internal/toolstream`。 +6. 响应格式：`internal/format/*` 或 `internal/translatorcliproxy`。 + +对话记录页面问题优先检查： + +- Admin API：`/admin/chat-history`、`/admin/chat-history/{id}`。 +- 后端存储：`internal/chathistory/store.go`。 +- 前端轮询和 ETag：`webui/src/features/chatHistory/ChatHistoryContainer.jsx`。 + +Tool call 问题优先跑： + +```bash +go test -v ./internal/toolcall ./internal/toolstream -count=1 +node --test tests/node/stream-tool-sieve.test.js tests/node/chat-stream.test.js +``` + +## 5. 测试选择 + +小范围 Go 改动： + +```bash +go test ./internal/ -count=1 +``` + +前端改动： + +```bash +npm run build --prefix webui +``` + +高风险协议或流式改动： + +```bash +./tests/scripts/run-unit-all.sh +``` + +发布或真实账号链路验证： + +```bash +./tests/scripts/run-live.sh +``` + +端到端测试产物默认写入 `artifacts/testsuite/`。分享日志前需要清理 token、密码、cookie 和原始请求响应内容。 diff --git a/docs/README.md b/docs/README.md index a80093c..b3556eb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,8 @@ 3. [接口文档（API）](../API.md) 4. [部署指南](./DEPLOY.md) 5. [测试指南](./TESTING.md) -6. [贡献指南](./CONTRIBUTING.md) +6. [开发者速查](./DEVELOPMENT.md) +7. [贡献指南](./CONTRIBUTING.md) ### 专题文档 @@ -41,7 +42,8 @@ Recommended reading order: 3. [API reference](../API.en.md) 4. [Deployment guide](./DEPLOY.en.md) 5. [Testing guide](./TESTING.md) -6. [Contributing guide](./CONTRIBUTING.en.md) +6. [Developer quick reference](./DEVELOPMENT.md) +7. [Contributing guide](./CONTRIBUTING.en.md) ### Topical docs diff --git a/internal/chathistory/store.go b/internal/chathistory/store.go index faa1818..8f215a1 100644 --- a/internal/chathistory/store.go +++ b/internal/chathistory/store.go @@ -192,6 +192,18 @@ func (s *Store) Snapshot() (File, error) { return cloneFile(s.state), nil } +func (s *Store) Revision() (int64, error) { + if s == nil { + return 0, errors.New("chat history store is nil") + } + s.mu.Lock() + defer s.mu.Unlock() + if s.err != nil { + return 0, s.err + } + return s.state.Revision, nil +} + func (s *Store) Enabled() bool { if s == nil { return false @@ -220,6 +232,22 @@ func (s *Store) Get(id string) (Entry, error) { return cloneEntry(item), nil } +func (s *Store) DetailRevision(id string) (int64, error) { + if s == nil { + return 0, errors.New("chat history store is nil") + } + s.mu.Lock() + defer s.mu.Unlock() + if s.err != nil { + return 0, s.err + } + item, ok := s.details[strings.TrimSpace(id)] + if !ok { + return 0, errors.New("chat history entry not found") + } + return item.Revision, nil +} + func (s *Store) Start(params StartParams) (Entry, error) { if s == nil { return Entry{}, errors.New("chat history store is nil") diff --git a/internal/httpapi/admin/history/handler_chat_history.go b/internal/httpapi/admin/history/handler_chat_history.go index e05a9e3..8072a2a 100644 --- a/internal/httpapi/admin/history/handler_chat_history.go +++ b/internal/httpapi/admin/history/handler_chat_history.go @@ -16,6 +16,24 @@ func (h *Handler) getChatHistory(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusServiceUnavailable, map[string]any{"detail": "chat history store is not configured"}) return } + ifNoneMatch := strings.TrimSpace(r.Header.Get("If-None-Match")) + if ifNoneMatch != "" { + revision, err := store.Revision() + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{ + "detail": err.Error(), + "path": store.Path(), + }) + return + } + etag := chathistory.ListETag(revision) + w.Header().Set("ETag", etag) + w.Header().Set("Cache-Control", "no-cache") + if ifNoneMatch == etag { + w.WriteHeader(http.StatusNotModified) + return + } + } snapshot, err := store.Snapshot() if err != nil { writeJSON(w, http.StatusServiceUnavailable, map[string]any{ @@ -27,7 +45,7 @@ func (h *Handler) getChatHistory(w http.ResponseWriter, r *http.Request) { etag := chathistory.ListETag(snapshot.Revision) w.Header().Set("ETag", etag) w.Header().Set("Cache-Control", "no-cache") - if strings.TrimSpace(r.Header.Get("If-None-Match")) == etag { + if ifNoneMatch == etag { w.WriteHeader(http.StatusNotModified) return } @@ -51,6 +69,25 @@ func (h *Handler) getChatHistoryItem(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]any{"detail": "history id is required"}) return } + ifNoneMatch := strings.TrimSpace(r.Header.Get("If-None-Match")) + if ifNoneMatch != "" { + revision, err := store.DetailRevision(id) + if err != nil { + status := http.StatusInternalServerError + if strings.Contains(strings.ToLower(err.Error()), "not found") { + status = http.StatusNotFound + } + writeJSON(w, status, map[string]any{"detail": err.Error()}) + return + } + etag := chathistory.DetailETag(id, revision) + w.Header().Set("ETag", etag) + w.Header().Set("Cache-Control", "no-cache") + if ifNoneMatch == etag { + w.WriteHeader(http.StatusNotModified) + return + } + } item, err := store.Get(id) if err != nil { status := http.StatusInternalServerError @@ -63,7 +100,7 @@ func (h *Handler) getChatHistoryItem(w http.ResponseWriter, r *http.Request) { etag := chathistory.DetailETag(item.ID, item.Revision) w.Header().Set("ETag", etag) w.Header().Set("Cache-Control", "no-cache") - if strings.TrimSpace(r.Header.Get("If-None-Match")) == etag { + if ifNoneMatch == etag { w.WriteHeader(http.StatusNotModified) return } diff --git a/internal/httpapi/admin/history/handler_chat_history_test.go b/internal/httpapi/admin/history/handler_chat_history_test.go index 1397bae..4d3e32f 100644 --- a/internal/httpapi/admin/history/handler_chat_history_test.go +++ b/internal/httpapi/admin/history/handler_chat_history_test.go @@ -95,6 +95,15 @@ func TestGetChatHistoryAndUpdateSettings(t *testing.T) { t.Fatalf("expected detail etag header") } + notModifiedItemReq := httptest.NewRequest(http.MethodGet, "/chat-history/"+entry.ID, nil) + notModifiedItemReq.Header.Set("Authorization", "Bearer admin") + notModifiedItemReq.Header.Set("If-None-Match", itemRec.Header().Get("ETag")) + notModifiedItemRec := httptest.NewRecorder() + r.ServeHTTP(notModifiedItemRec, notModifiedItemReq) + if notModifiedItemRec.Code != http.StatusNotModified { + t.Fatalf("expected detail 304, got %d body=%s", notModifiedItemRec.Code, notModifiedItemRec.Body.String()) + } + updateReq := httptest.NewRequest(http.MethodPut, "/chat-history/settings", bytes.NewReader([]byte(`{"limit":10}`))) updateReq.Header.Set("Authorization", "Bearer admin") updateRec := httptest.NewRecorder() diff --git a/internal/toolstream/tool_sieve_xml.go b/internal/toolstream/tool_sieve_xml.go index 67d6555..b755200 100644 --- a/internal/toolstream/tool_sieve_xml.go +++ b/internal/toolstream/tool_sieve_xml.go @@ -2,50 +2,9 @@ package toolstream import ( "ds2api/internal/toolcall" - "regexp" "strings" ) -// --- XML tool call support for the streaming sieve --- - -//nolint:unused // kept as explicit tag inventory for future XML sieve refinements. -var xmlToolCallClosingTags = []string{"", "", "", "", ""} -var xmlToolCallOpeningTags = []string{ - ""}, - {""}, - {"<｜tool_calls", ""}, - {"<|tool_calls", ""}, - {""}, -} - -// xmlToolCallBlockPattern matches a complete canonical XML tool call block. -// -//nolint:unused // reserved for future fast-path XML block detection. -var xmlToolCallBlockPattern = regexp.MustCompile(`(?is)((?:]*>\s*(?:.*?)\s*(?:|))`) - -// xmlToolTagsToDetect is the set of XML tag prefixes used by findToolSegmentStart. -var xmlToolTagsToDetect = []string{ - "<|dsml|tool_calls>", "<|dsml|tool_calls\n", "<|dsml|tool_calls ", - "<|dsml|invoke ", "<|dsml|invoke\n", "<|dsml|invoke\t", "<|dsml|invoke\r", - "", "", "<｜tool_calls\n", "<｜tool_calls ", - "<｜invoke ", "<｜invoke\n", "<｜invoke\t", "<｜invoke\r", - "<|tool_calls>", "<|tool_calls\n", "<|tool_calls ", - "<|invoke ", "<|invoke\n", "<|invoke\t", "<|invoke\r", - "", "") - if end < 0 { - return -1 - } - i += len("") - case strings.HasPrefix(lower[i:], "") - if end < 0 { - return -1 - } - i += len("") - case strings.HasPrefix(lower[i:], closeTarget): - depth-- - if depth == 0 { - return i - } - i += len(closeTarget) - case strings.HasPrefix(lower[i:], openTarget) && hasXMLToolTagBoundary(s, i+len(openTarget)): - depth++ - i += len(openTarget) - default: - i++ - } - } - return -1 -} - -func findXMLOpenOutsideCDATA(s, openTag string, start int) int { - if s == "" || openTag == "" { - return -1 - } - if start < 0 { - start = 0 - } - lower := strings.ToLower(s) - target := strings.ToLower(openTag) - for i := start; i < len(s); { - switch { - case strings.HasPrefix(lower[i:], "") - if end < 0 { - return -1 - } - i += len("") - case strings.HasPrefix(lower[i:], "") - if end < 0 { - return -1 - } - i += len("") - case strings.HasPrefix(lower[i:], target) && hasXMLToolTagBoundary(s, i+len(target)): - return i - default: - i++ - } - } - return -1 -} - -func hasXMLToolTagBoundary(text string, idx int) bool { - if idx >= len(text) { - return true - } - switch text[idx] { - case ' ', '\t', '\n', '\r', '>', '/': - return true - default: - return false - } -} - // hasOpenXMLToolTag returns true if captured text contains an XML tool opening tag // whose SPECIFIC closing tag has not appeared yet. func hasOpenXMLToolTag(captured string) bool { @@ -307,59 +184,6 @@ func firstInvokeIndex(lower string) (int, bool) { } } -func findXMLCloseOutsideCDATA(s, closeTag string, start int) int { - if s == "" || closeTag == "" { - return -1 - } - if start < 0 { - start = 0 - } - lower := strings.ToLower(s) - target := strings.ToLower(closeTag) - for i := start; i < len(s); { - switch { - case strings.HasPrefix(lower[i:], "") - if end < 0 { - return -1 - } - i += len("") - case strings.HasPrefix(lower[i:], "") - if end < 0 { - return -1 - } - i += len("") - case strings.HasPrefix(lower[i:], target): - return i - default: - i++ - } - } - return -1 -} - -func findXMLTagEnd(s string, start int) int { - quote := byte(0) - for i := start; i < len(s); i++ { - ch := s[i] - if quote != 0 { - if ch == quote { - quote = 0 - } - continue - } - if ch == '"' || ch == '\'' { - quote = ch - continue - } - if ch == '>' { - return i - } - } - return -1 -} - // findPartialXMLToolTagStart checks if the string ends with a partial canonical // XML wrapper tag (e.g., "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], closeTarget): + depth-- + if depth == 0 { + return i + } + i += len(closeTarget) + case strings.HasPrefix(lower[i:], openTarget) && hasXMLToolTagBoundary(s, i+len(openTarget)): + depth++ + i += len(openTarget) + default: + i++ + } + } + return -1 +} + +func findXMLOpenOutsideCDATA(s, openTag string, start int) int { + if s == "" || openTag == "" { + return -1 + } + if start < 0 { + start = 0 + } + lower := strings.ToLower(s) + target := strings.ToLower(openTag) + for i := start; i < len(s); { + switch { + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], target) && hasXMLToolTagBoundary(s, i+len(target)): + return i + default: + i++ + } + } + return -1 +} + +func findXMLCloseOutsideCDATA(s, closeTag string, start int) int { + if s == "" || closeTag == "" { + return -1 + } + if start < 0 { + start = 0 + } + lower := strings.ToLower(s) + target := strings.ToLower(closeTag) + for i := start; i < len(s); { + switch { + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], "") + if end < 0 { + return -1 + } + i += len("") + case strings.HasPrefix(lower[i:], target): + return i + default: + i++ + } + } + return -1 +} + +func hasXMLToolTagBoundary(text string, idx int) bool { + if idx >= len(text) { + return true + } + switch text[idx] { + case ' ', '\t', '\n', '\r', '>', '/': + return true + default: + return false + } +} + +func findXMLTagEnd(s string, start int) int { + quote := byte(0) + for i := start; i < len(s); i++ { + ch := s[i] + if quote != 0 { + if ch == quote { + quote = 0 + } + continue + } + if ch == '"' || ch == '\'' { + quote = ch + continue + } + if ch == '>' { + return i + } + } + return -1 +} diff --git a/internal/toolstream/tool_sieve_xml_tags.go b/internal/toolstream/tool_sieve_xml_tags.go new file mode 100644 index 0000000..f345354 --- /dev/null +++ b/internal/toolstream/tool_sieve_xml_tags.go @@ -0,0 +1,43 @@ +package toolstream + +import "regexp" + +// --- XML tool call support for the streaming sieve --- + +//nolint:unused // kept as explicit tag inventory for future XML sieve refinements. +var xmlToolCallClosingTags = []string{"", "", "", "", ""} +var xmlToolCallOpeningTags = []string{ + ""}, + {""}, + {"<｜tool_calls", ""}, + {"<|tool_calls", ""}, + {""}, +} + +// xmlToolCallBlockPattern matches a complete canonical XML tool call block. +// +//nolint:unused // reserved for future fast-path XML block detection. +var xmlToolCallBlockPattern = regexp.MustCompile(`(?is)((?:]*>\s*(?:.*?)\s*(?:|))`) + +// xmlToolTagsToDetect is the set of XML tag prefixes used by findToolSegmentStart. +var xmlToolTagsToDetect = []string{ + "<|dsml|tool_calls>", "<|dsml|tool_calls\n", "<|dsml|tool_calls ", + "<|dsml|invoke ", "<|dsml|invoke\n", "<|dsml|invoke\t", "<|dsml|invoke\r", + "", "", "<｜tool_calls\n", "<｜tool_calls ", + "<｜invoke ", "<｜invoke\n", "<｜invoke\t", "<｜invoke\r", + "<|tool_calls>", "<|tool_calls\n", "<|tool_calls ", + "<|invoke ", "<|invoke\n", "<|invoke\t", "<|invoke\r", + "", " import('../features/account/AccountManagerContainer')) +const ApiTesterContainer = lazy(() => import('../features/apiTester/ApiTesterContainer')) +const ChatHistoryContainer = lazy(() => import('../features/chatHistory/ChatHistoryContainer')) +const BatchImport = lazy(() => import('../components/BatchImport')) +const VercelSyncContainer = lazy(() => import('../features/vercel/VercelSyncContainer')) +const SettingsContainer = lazy(() => import('../features/settings/SettingsContainer')) +const ProxyManagerContainer = lazy(() => import('../features/proxy/ProxyManagerContainer')) + +function TabLoadingFallback({ label }) { + return ( +
+
+ + {label} +
+
+ ) +} + export default function DashboardShell({ token, onLogout, config, fetchConfig, showMessage, message, onForceLogout, isVercel }) { const { t } = useI18n() const location = useLocation() @@ -47,6 +60,7 @@ export default function DashboardShell({ token, onLogout, config, fetchConfig, s const pathTab = routeSegments[0] || '' const activeTab = tabIds.has(pathTab) ? pathTab : 'accounts' const adminBasePath = pathSegments[0] === 'admin' ? '/admin' : '' + const activeNavItem = navItems.find(n => n.id === activeTab) const navigateToTab = useCallback((tabID) => { const nextPath = tabID === 'accounts' @@ -232,10 +246,10 @@ export default function DashboardShell({ token, onLogout, config, fetchConfig, s

- {navItems.find(n => n.id === activeTab)?.label} + {activeNavItem?.label}

- {navItems.find(n => n.id === activeTab)?.description} + {activeNavItem?.description}

@@ -251,7 +265,9 @@ export default function DashboardShell({ token, onLogout, config, fetchConfig, s )}
- {renderTab()} + }> + {renderTab()} +