From 8cbb5a4262ea7fbaabe1d3075408e5f464366b32 Mon Sep 17 00:00:00 2001 From: CJACK Date: Tue, 17 Feb 2026 02:03:41 +0800 Subject: [PATCH] feat: Introduce a new Go-based test suite runner with supporting scripts and documentation. --- .gitignore | 1 + DEPLOY.en.md | 26 + DEPLOY.md | 26 + README.MD | 18 + README.en.md | 18 + TESTING.md | 95 ++ cmd/ds2api-tests/main.go | 36 + internal/testsuite/runner.go | 1564 +++++++++++++++++++++++++++++++++ scripts/testsuite/run-live.sh | 8 + 9 files changed, 1792 insertions(+) create mode 100644 TESTING.md create mode 100644 cmd/ds2api-tests/main.go create mode 100644 internal/testsuite/runner.go create mode 100755 scripts/testsuite/run-live.sh diff --git a/.gitignore b/.gitignore index 8f7ada8..bace4b7 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ env/ *.log logs/ uvicorn.log +artifacts/ # Vercel .vercel diff --git a/DEPLOY.en.md b/DEPLOY.en.md index 2d88c27..1c4fa7b 100644 --- a/DEPLOY.en.md +++ b/DEPLOY.en.md @@ -225,3 +225,29 @@ If admin UI is required: ```bash curl -s http://127.0.0.1:5001/admin ``` + +## 7. Pre-release Local Regression (Recommended) + +Run the full live testsuite before release: + +```bash +./scripts/testsuite/run-live.sh +``` + +Optional flags: + +```bash +go run ./cmd/ds2api-tests \ + --config config.json \ + --admin-key admin \ + --out artifacts/testsuite \ + --timeout 120 \ + --retries 2 +``` + +The testsuite automatically performs: + +- preflight checks (syntax/build/unit tests) +- isolated config copy startup (no mutation to your original `config.json`) +- live scenario verification (OpenAI/Claude/Admin/concurrency/toolcall/streaming) +- full request/response artifact logging for debugging diff --git a/DEPLOY.md b/DEPLOY.md index cb2c7b5..8ba9efe 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -223,3 +223,29 @@ curl -s http://127.0.0.1:5001/v1/models ```bash curl -s http://127.0.0.1:5001/admin ``` + +## 7. 发布前本地回归（推荐） + +建议在发布前执行一次完整测试集（真实账号链路）： + +```bash +./scripts/testsuite/run-live.sh +``` + +可选参数示例： + +```bash +go run ./cmd/ds2api-tests \ + --config config.json \ + --admin-key admin \ + --out artifacts/testsuite \ + --timeout 120 \ + --retries 2 +``` + +测试集会自动执行： + +- 语法/构建/单测 preflight +- 隔离副本配置启动服务（不污染原始 `config.json`） +- 真实调用场景验证（OpenAI/Claude/Admin/并发/toolcall/流式） +- 全量请求与响应日志落盘（用于故障复盘） diff --git a/README.MD b/README.MD index d21854a..c8a2018 100644 --- a/README.MD +++ b/README.MD @@ -210,11 +210,29 @@ cp config.example.json config.json - API 文档：`API.md` / `API.en.md` - 部署文档：`DEPLOY.md` / `DEPLOY.en.md` - 贡献指南：`CONTRIBUTING.md` / `CONTRIBUTING.en.md` +- 测试集文档：`TESTING.md` ```bash go test ./... ``` +一键真实账号全链路测试（会生成完整请求/响应日志）： + +```bash +./scripts/testsuite/run-live.sh +``` + +或使用可配置参数： + +```bash +go run ./cmd/ds2api-tests \ + --config config.json \ + --admin-key admin \ + --out artifacts/testsuite \ + --timeout 120 \ + --retries 2 +``` + ## 免责声明 本项目基于逆向方式实现，仅供学习与研究使用。稳定性和可用性不作保证，请勿用于违反服务条款或法律法规的场景。 diff --git a/README.en.md b/README.en.md index 26e82fb..bc0c7e2 100644 --- a/README.en.md +++ b/README.en.md @@ -210,11 +210,29 @@ Tool-call leakage is handled in the current implementation: - API docs: `API.md` / `API.en.md` - Deployment docs: `DEPLOY.md` / `DEPLOY.en.md` - Contributing: `CONTRIBUTING.md` / `CONTRIBUTING.en.md` +- Testsuite guide: `TESTING.md` ```bash go test ./... ``` +One-command live end-to-end tests (with full request/response logs): + +```bash +./scripts/testsuite/run-live.sh +``` + +Or run with explicit flags: + +```bash +go run ./cmd/ds2api-tests \ + --config config.json \ + --admin-key admin \ + --out artifacts/testsuite \ + --timeout 120 \ + --retries 2 +``` + ## Disclaimer This project is built through reverse engineering and is provided for learning and research only. Stability is not guaranteed. Do not use it in scenarios that violate terms of service or laws. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..beee773 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,95 @@ +# DS2API Testing Guide + +## Overview + +DS2API provides a live end-to-end testsuite that runs against your **local configured accounts** and records full artifacts for post-mortem debugging. + +Entry points: + +- `./scripts/testsuite/run-live.sh` +- `go run ./cmd/ds2api-tests` + +## Quick Start + +```bash +./scripts/testsuite/run-live.sh +``` + +Default behavior: + +- runs preflight checks: + - `go test ./... -count=1` + - `node --check api/chat-stream.js` + - `node --check api/helpers/stream-tool-sieve.js` + - `npm run build --prefix webui` +- copies `config.json` into an isolated temporary config +- starts local server with `go run ./cmd/ds2api` +- executes live scenarios (OpenAI/Claude/Admin/stream/toolcall/concurrency) +- continues on failures and writes final summary + +## CLI Flags + +```bash +go run ./cmd/ds2api-tests \ + --config config.json \ + --admin-key admin \ + --out artifacts/testsuite \ + --port 0 \ + --timeout 120 \ + --retries 2 \ + --no-preflight=false +``` + +- `--config`: config file path (default `config.json`) +- `--admin-key`: admin key (default from `DS2API_ADMIN_KEY`, fallback `admin`) +- `--out`: artifact root directory (default `artifacts/testsuite`) +- `--port`: test server port (`0` = auto pick free port) +- `--timeout`: per request timeout in seconds (default `120`) +- `--retries`: retry count for network/5xx requests (default `2`) +- `--no-preflight`: skip preflight checks + +## Artifact Layout + +Each run creates: + +`artifacts/testsuite//` + +- `summary.json`: machine-readable report +- `summary.md`: human-readable report +- `server.log`: server stdout/stderr log during run +- `preflight.log`: preflight command outputs +- `cases//` + - `request.json` + - `response.headers` + - `response.body` + - `stream.raw` + - `assertions.json` + - `meta.json` + +## Trace Binding (for fast debugging) + +Each request includes: + +- header: `X-Ds2-Test-Trace: ` +- query: `__trace_id=` + +When a case fails, `summary.md` includes trace IDs. You can locate related server logs quickly: + +```bash +rg "" artifacts/testsuite//server.log +``` + +## Exit Code + +- `0`: all cases passed +- `1`: one or more cases failed + +This allows using the testsuite as a local release gate. + +## Sensitive Data Warning + +This testsuite stores **full raw request/response payloads** for debugging. + +- Do not upload artifacts publicly. +- Do not share artifact directories in issue trackers without manual redaction. + diff --git a/cmd/ds2api-tests/main.go b/cmd/ds2api-tests/main.go new file mode 100644 index 0000000..5966cf0 --- /dev/null +++ b/cmd/ds2api-tests/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + "ds2api/internal/testsuite" +) + +func main() { + opts := testsuite.DefaultOptions() + var timeoutSeconds int + + flag.StringVar(&opts.ConfigPath, "config", opts.ConfigPath, "Path to config file (default: config.json)") + flag.StringVar(&opts.AdminKey, "admin-key", opts.AdminKey, "Admin key (default: DS2API_ADMIN_KEY or admin)") + flag.StringVar(&opts.OutputDir, "out", opts.OutputDir, "Output artifact directory") + flag.IntVar(&opts.Port, "port", opts.Port, "Server port (0 means auto-select free port)") + flag.IntVar(&timeoutSeconds, "timeout", int(opts.Timeout.Seconds()), "Per-request timeout in seconds") + flag.IntVar(&opts.Retries, "retries", opts.Retries, "Retry count for network/5xx requests") + flag.BoolVar(&opts.NoPreflight, "no-preflight", opts.NoPreflight, "Skip preflight checks") + flag.Parse() + + if timeoutSeconds <= 0 { + timeoutSeconds = 120 + } + opts.Timeout = time.Duration(timeoutSeconds) * time.Second + + if err := testsuite.Run(context.Background(), opts); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + fmt.Fprintln(os.Stdout, "testsuite completed successfully") +} diff --git a/internal/testsuite/runner.go b/internal/testsuite/runner.go new file mode 100644 index 0000000..3a0f297 --- /dev/null +++ b/internal/testsuite/runner.go @@ -0,0 +1,1564 @@ +package testsuite + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type Options struct { + ConfigPath string + AdminKey string + OutputDir string + Port int + Timeout time.Duration + Retries int + NoPreflight bool +} + +type runSummary struct { + RunID string `json:"run_id"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at"` + DurationMS int64 `json:"duration_ms"` + Stats map[string]any `json:"stats"` + Environment map[string]any `json:"environment"` + Cases []caseResult `json:"cases"` + Warnings []string `json:"warnings,omitempty"` +} + +type caseResult struct { + CaseID string `json:"case_id"` + Passed bool `json:"passed"` + DurationMS int64 `json:"duration_ms"` + TraceIDs []string `json:"trace_ids"` + StatusCodes []int `json:"status_codes"` + Error string `json:"error,omitempty"` + ArtifactPath string `json:"artifact_path"` + Assertions []assertionResult `json:"assertions"` +} + +type assertionResult struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Detail string `json:"detail,omitempty"` +} + +type requestLog struct { + Seq int `json:"seq"` + Attempt int `json:"attempt"` + TraceID string `json:"trace_id"` + Method string `json:"method"` + URL string `json:"url"` + Headers map[string]string `json:"headers"` + Body any `json:"body,omitempty"` + Timestamp string `json:"timestamp"` +} + +type responseLog struct { + Seq int `json:"seq"` + Attempt int `json:"attempt"` + TraceID string `json:"trace_id"` + StatusCode int `json:"status_code"` + Headers map[string][]string `json:"headers"` + BodyText string `json:"body_text"` + DurationMS int64 `json:"duration_ms"` + NetworkErr string `json:"network_error,omitempty"` + ReceivedAt string `json:"received_at"` +} + +type caseContext struct { + runner *Runner + id string + dir string + startedAt time.Time + seq int + assertions []assertionResult + requests []requestLog + responses []responseLog + streamRaw strings.Builder + traceIDsSet map[string]struct{} +} + +type requestSpec struct { + Method string + Path string + Headers map[string]string + Body any + Stream bool + Retryable bool +} + +type responseResult struct { + StatusCode int + Headers http.Header + Body []byte + TraceID string + URL string +} + +type Runner struct { + opts Options + + runID string + runDir string + serverLog string + preflightLog string + + baseURL string + httpClient *http.Client + serverCmd *exec.Cmd + serverLogFd *os.File + + configCopyPath string + originalConfigPath string + originalConfigHash string + + configRaw runConfig + apiKey string + adminKey string + adminJWT string + accountID string + + warnings []string + results []caseResult +} + +type runConfig struct { + Keys []string `json:"keys"` + Accounts []struct { + Email string `json:"email,omitempty"` + Mobile string `json:"mobile,omitempty"` + } `json:"accounts"` +} + +func DefaultOptions() Options { + return Options{ + ConfigPath: "config.json", + AdminKey: strings.TrimSpace(os.Getenv("DS2API_ADMIN_KEY")), + OutputDir: "artifacts/testsuite", + Port: 0, + Timeout: 120 * time.Second, + Retries: 2, + NoPreflight: false, + } +} + +func Run(ctx context.Context, opts Options) error { + r, err := newRunner(opts) + if err != nil { + return err + } + start := time.Now() + defer func() { + _ = r.stopServer() + }() + + if err := r.prepareRunDir(); err != nil { + return err + } + + if !r.opts.NoPreflight { + if err := r.runPreflight(ctx); err != nil { + _ = r.writeSummary(start, time.Now()) + return err + } + } + + if err := r.prepareConfigIsolation(); err != nil { + _ = r.writeSummary(start, time.Now()) + return err + } + + if err := r.startServer(ctx); err != nil { + _ = r.writeSummary(start, time.Now()) + return err + } + + if err := r.prepareAuth(ctx); err != nil { + r.warnings = append(r.warnings, "auth prepare failed: "+err.Error()) + } + + for _, c := range r.cases() { + r.runCase(ctx, c) + } + + if err := r.ensureOriginalConfigUntouched(); err != nil { + r.warnings = append(r.warnings, err.Error()) + } + + end := time.Now() + if err := r.writeSummary(start, end); err != nil { + return err + } + + failed := 0 + for _, cs := range r.results { + if !cs.Passed { + failed++ + } + } + if failed > 0 { + return fmt.Errorf("testsuite failed: %d case(s) failed, see %s", failed, filepath.Join(r.runDir, "summary.md")) + } + return nil +} + +func newRunner(opts Options) (*Runner, error) { + if strings.TrimSpace(opts.ConfigPath) == "" { + opts.ConfigPath = "config.json" + } + if strings.TrimSpace(opts.OutputDir) == "" { + opts.OutputDir = "artifacts/testsuite" + } + if opts.Timeout <= 0 { + opts.Timeout = 120 * time.Second + } + if opts.Retries < 0 { + opts.Retries = 0 + } + adminKey := strings.TrimSpace(opts.AdminKey) + if adminKey == "" { + adminKey = strings.TrimSpace(os.Getenv("DS2API_ADMIN_KEY")) + } + if adminKey == "" { + adminKey = "admin" + } + opts.AdminKey = adminKey + + return &Runner{ + opts: opts, + httpClient: &http.Client{ + Timeout: 0, + }, + runID: time.Now().UTC().Format("20060102T150405Z"), + adminKey: adminKey, + }, nil +} + +func (r *Runner) prepareRunDir() error { + r.runDir = filepath.Join(r.opts.OutputDir, r.runID) + if err := os.MkdirAll(r.runDir, 0o755); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(r.runDir, "cases"), 0o755); err != nil { + return err + } + r.serverLog = filepath.Join(r.runDir, "server.log") + r.preflightLog = filepath.Join(r.runDir, "preflight.log") + return nil +} + +func (r *Runner) runPreflight(ctx context.Context) error { + steps := [][]string{ + {"go", "test", "./...", "-count=1"}, + {"node", "--check", "api/chat-stream.js"}, + {"node", "--check", "api/helpers/stream-tool-sieve.js"}, + {"npm", "run", "build", "--prefix", "webui"}, + } + f, err := os.OpenFile(r.preflightLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return err + } + defer f.Close() + for _, step := range steps { + if _, err := fmt.Fprintf(f, "\n$ %s\n", strings.Join(step, " ")); err != nil { + return err + } + cmd := exec.CommandContext(ctx, step[0], step[1:]...) + cmd.Stdout = f + cmd.Stderr = f + if err := cmd.Run(); err != nil { + return fmt.Errorf("preflight failed at `%s`: %w", strings.Join(step, " "), err) + } + } + return nil +} + +func (r *Runner) prepareConfigIsolation() error { + abs, err := filepath.Abs(r.opts.ConfigPath) + if err != nil { + return err + } + r.originalConfigPath = abs + raw, err := os.ReadFile(abs) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + r.originalConfigHash = hex.EncodeToString(sum[:]) + + tmpDir := filepath.Join(r.runDir, "tmp") + if err := os.MkdirAll(tmpDir, 0o755); err != nil { + return err + } + r.configCopyPath = filepath.Join(tmpDir, "config.json") + if err := os.WriteFile(r.configCopyPath, raw, 0o644); err != nil { + return err + } + var cfg runConfig + if err := json.Unmarshal(raw, &cfg); err != nil { + return fmt.Errorf("parse config failed: %w", err) + } + r.configRaw = cfg + if len(cfg.Keys) > 0 { + r.apiKey = strings.TrimSpace(cfg.Keys[0]) + } + for _, acc := range cfg.Accounts { + id := strings.TrimSpace(acc.Email) + if id == "" { + id = strings.TrimSpace(acc.Mobile) + } + if id != "" { + r.accountID = id + break + } + } + return nil +} + +func (r *Runner) startServer(ctx context.Context) error { + port := r.opts.Port + if port <= 0 { + p, err := findFreePort() + if err != nil { + return err + } + port = p + } + r.baseURL = "http://127.0.0.1:" + strconv.Itoa(port) + + logFd, err := os.OpenFile(r.serverLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return err + } + r.serverLogFd = logFd + cmd := exec.CommandContext(ctx, "go", "run", "./cmd/ds2api") + cmd.Stdout = logFd + cmd.Stderr = logFd + cmd.Env = prepareServerEnv(os.Environ(), map[string]string{ + "PORT": strconv.Itoa(port), + "DS2API_CONFIG_PATH": r.configCopyPath, + "DS2API_AUTO_BUILD_WEBUI": "false", + "DS2API_CONFIG_JSON": "", + "CONFIG_JSON": "", + }) + if err := cmd.Start(); err != nil { + _ = logFd.Close() + return err + } + r.serverCmd = cmd + + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + if r.ping("/healthz") == nil && r.ping("/readyz") == nil { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return errors.New("server readiness timeout") +} + +func (r *Runner) stopServer() error { + var errs []string + if r.serverCmd != nil && r.serverCmd.Process != nil { + _ = r.serverCmd.Process.Signal(os.Interrupt) + done := make(chan error, 1) + go func() { done <- r.serverCmd.Wait() }() + select { + case <-time.After(5 * time.Second): + _ = r.serverCmd.Process.Kill() + <-done + case <-done: + } + } + if r.serverLogFd != nil { + if err := r.serverLogFd.Close(); err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return errors.New(strings.Join(errs, "; ")) + } + return nil +} + +func (r *Runner) ping(path string) error { + resp, err := r.httpClient.Get(r.baseURL + path) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status=%d", resp.StatusCode) + } + return nil +} + +func (r *Runner) prepareAuth(ctx context.Context) error { + reqBody := map[string]any{ + "admin_key": r.adminKey, + "expire_hours": 24, + } + resp, err := r.doSimpleJSON(ctx, http.MethodPost, "/admin/login", nil, reqBody) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("admin login status=%d body=%s", resp.StatusCode, string(resp.Body)) + } + var m map[string]any + if err := json.Unmarshal(resp.Body, &m); err != nil { + return err + } + token, _ := m["token"].(string) + if strings.TrimSpace(token) == "" { + return errors.New("empty admin jwt token") + } + r.adminJWT = token + return nil +} + +func (r *Runner) ensureOriginalConfigUntouched() error { + raw, err := os.ReadFile(r.originalConfigPath) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + current := hex.EncodeToString(sum[:]) + if current != r.originalConfigHash { + return fmt.Errorf("original config changed unexpectedly: %s", r.originalConfigPath) + } + return nil +} + +func (r *Runner) runCase(ctx context.Context, c caseDef) { + caseDir := filepath.Join(r.runDir, "cases", c.ID) + _ = os.MkdirAll(caseDir, 0o755) + cc := &caseContext{ + runner: r, + id: c.ID, + dir: caseDir, + startedAt: time.Now(), + traceIDsSet: map[string]struct{}{}, + } + err := c.Run(ctx, cc) + duration := time.Since(cc.startedAt).Milliseconds() + + if err != nil { + cc.assertions = append(cc.assertions, assertionResult{ + Name: "case_error", + Passed: false, + Detail: err.Error(), + }) + } + passed := err == nil + for _, a := range cc.assertions { + if !a.Passed { + passed = false + break + } + } + + traceIDs := make([]string, 0, len(cc.traceIDsSet)) + for t := range cc.traceIDsSet { + traceIDs = append(traceIDs, t) + } + sort.Strings(traceIDs) + statuses := uniqueStatusCodes(cc.responses) + cs := caseResult{ + CaseID: c.ID, + Passed: passed, + DurationMS: duration, + TraceIDs: traceIDs, + StatusCodes: statuses, + ArtifactPath: caseDir, + Assertions: cc.assertions, + } + if err != nil { + cs.Error = err.Error() + } + _ = cc.flushArtifacts(cs) + r.results = append(r.results, cs) +} + +func (cc *caseContext) assert(name string, ok bool, detail string) { + cc.assertions = append(cc.assertions, assertionResult{ + Name: name, + Passed: ok, + Detail: detail, + }) +} + +func (cc *caseContext) request(ctx context.Context, spec requestSpec) (*responseResult, error) { + retries := cc.runner.opts.Retries + if !spec.Retryable { + retries = 0 + } + var lastErr error + for attempt := 1; attempt <= retries+1; attempt++ { + resp, err := cc.requestOnce(ctx, spec, attempt) + if err == nil && resp.StatusCode < 500 { + return resp, nil + } + if err != nil { + lastErr = err + } else if resp.StatusCode >= 500 { + lastErr = fmt.Errorf("status=%d", resp.StatusCode) + } + if attempt <= retries { + sleep := time.Duration(300*(1<<(attempt-1))) * time.Millisecond + time.Sleep(sleep) + } + } + return nil, lastErr +} + +func (cc *caseContext) requestOnce(ctx context.Context, spec requestSpec, attempt int) (*responseResult, error) { + cc.seq++ + traceID := fmt.Sprintf("ts_%s_%s_%03d", cc.runner.runID, sanitizeID(cc.id), cc.seq) + cc.traceIDsSet[traceID] = struct{}{} + + fullURL, err := withTraceQuery(cc.runner.baseURL+spec.Path, traceID) + if err != nil { + return nil, err + } + + headers := map[string]string{} + for k, v := range spec.Headers { + headers[k] = v + } + headers["X-Ds2-Test-Trace"] = traceID + + var bodyBytes []byte + var bodyAny any + if spec.Body != nil { + b, err := json.Marshal(spec.Body) + if err != nil { + return nil, err + } + bodyBytes = b + bodyAny = spec.Body + headers["Content-Type"] = "application/json" + } + cc.requests = append(cc.requests, requestLog{ + Seq: cc.seq, + Attempt: attempt, + TraceID: traceID, + Method: spec.Method, + URL: fullURL, + Headers: headers, + Body: bodyAny, + Timestamp: time.Now().Format(time.RFC3339Nano), + }) + + reqCtx, cancel := context.WithTimeout(ctx, cc.runner.opts.Timeout) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, spec.Method, fullURL, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, err + } + for k, v := range headers { + req.Header.Set(k, v) + } + start := time.Now() + resp, err := cc.runner.httpClient.Do(req) + if err != nil { + cc.responses = append(cc.responses, responseLog{ + Seq: cc.seq, + Attempt: attempt, + TraceID: traceID, + StatusCode: 0, + DurationMS: time.Since(start).Milliseconds(), + NetworkErr: err.Error(), + ReceivedAt: time.Now().Format(time.RFC3339Nano), + }) + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + cc.responses = append(cc.responses, responseLog{ + Seq: cc.seq, + Attempt: attempt, + TraceID: traceID, + StatusCode: resp.StatusCode, + Headers: resp.Header, + BodyText: string(body), + DurationMS: time.Since(start).Milliseconds(), + ReceivedAt: time.Now().Format(time.RFC3339Nano), + }) + + if spec.Stream { + cc.streamRaw.WriteString(fmt.Sprintf("### trace=%s url=%s\n", traceID, fullURL)) + cc.streamRaw.Write(body) + cc.streamRaw.WriteString("\n\n") + } + + return &responseResult{ + StatusCode: resp.StatusCode, + Headers: resp.Header, + Body: body, + TraceID: traceID, + URL: fullURL, + }, nil +} + +func (cc *caseContext) flushArtifacts(cs caseResult) error { + requestPath := filepath.Join(cc.dir, "request.json") + headersPath := filepath.Join(cc.dir, "response.headers") + bodyPath := filepath.Join(cc.dir, "response.body") + streamPath := filepath.Join(cc.dir, "stream.raw") + assertPath := filepath.Join(cc.dir, "assertions.json") + metaPath := filepath.Join(cc.dir, "meta.json") + + if err := writeJSONFile(requestPath, cc.requests); err != nil { + return err + } + respHeaders := make([]map[string]any, 0, len(cc.responses)) + respBodies := make([]map[string]any, 0, len(cc.responses)) + for _, r := range cc.responses { + respHeaders = append(respHeaders, map[string]any{ + "seq": r.Seq, + "attempt": r.Attempt, + "trace_id": r.TraceID, + "status_code": r.StatusCode, + "headers": r.Headers, + }) + respBodies = append(respBodies, map[string]any{ + "seq": r.Seq, + "attempt": r.Attempt, + "trace_id": r.TraceID, + "status_code": r.StatusCode, + "body_text": r.BodyText, + "network_error": r.NetworkErr, + "duration_ms": r.DurationMS, + }) + } + if err := writeJSONFile(headersPath, respHeaders); err != nil { + return err + } + if err := writeJSONFile(bodyPath, respBodies); err != nil { + return err + } + if err := os.WriteFile(streamPath, []byte(cc.streamRaw.String()), 0o644); err != nil { + return err + } + if err := writeJSONFile(assertPath, cc.assertions); err != nil { + return err + } + meta := map[string]any{ + "case_id": cs.CaseID, + "trace_id": strings.Join(cs.TraceIDs, ","), + "attempt": len(cc.responses), + "duration_ms": cs.DurationMS, + "status": map[bool]string{true: "passed", false: "failed"}[cs.Passed], + "status_codes": cs.StatusCodes, + "assertions": cs.Assertions, + "artifact_path": cs.ArtifactPath, + } + return writeJSONFile(metaPath, meta) +} + +type caseDef struct { + ID string + Run func(context.Context, *caseContext) error +} + +func (r *Runner) cases() []caseDef { + return []caseDef{ + {ID: "healthz_ok", Run: r.caseHealthz}, + {ID: "readyz_ok", Run: r.caseReadyz}, + {ID: "models_openai", Run: r.caseModelsOpenAI}, + {ID: "models_claude", Run: r.caseModelsClaude}, + {ID: "admin_login_verify", Run: r.caseAdminLoginVerify}, + {ID: "admin_queue_status", Run: r.caseAdminQueueStatus}, + {ID: "chat_nonstream_basic", Run: r.caseChatNonstream}, + {ID: "chat_stream_basic", Run: r.caseChatStream}, + {ID: "reasoner_stream", Run: r.caseReasonerStream}, + {ID: "toolcall_nonstream", Run: r.caseToolcallNonstream}, + {ID: "toolcall_stream", Run: r.caseToolcallStream}, + {ID: "anthropic_messages_nonstream", Run: r.caseAnthropicNonstream}, + {ID: "anthropic_messages_stream", Run: r.caseAnthropicStream}, + {ID: "anthropic_count_tokens", Run: r.caseAnthropicCountTokens}, + {ID: "admin_account_test_single", Run: r.caseAdminAccountTest}, + {ID: "concurrency_burst", Run: r.caseConcurrencyBurst}, + {ID: "config_write_isolated", Run: r.caseConfigWriteIsolated}, + {ID: "error_contract_invalid_key", Run: r.caseInvalidKey}, + } +} + +func (r *Runner) caseHealthz(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{Method: http.MethodGet, Path: "/healthz", Retryable: true}) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + cc.assert("status_ok", asString(m["status"]) == "ok", fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseReadyz(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{Method: http.MethodGet, Path: "/readyz", Retryable: true}) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + cc.assert("status_ready", asString(m["status"]) == "ready", fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseModelsOpenAI(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{Method: http.MethodGet, Path: "/v1/models", Retryable: true}) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + ids := extractModelIDs(resp.Body) + cc.assert("has_deepseek_chat", contains(ids, "deepseek-chat"), strings.Join(ids, ",")) + cc.assert("has_deepseek_reasoner", contains(ids, "deepseek-reasoner"), strings.Join(ids, ",")) + return nil +} + +func (r *Runner) caseModelsClaude(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{Method: http.MethodGet, Path: "/anthropic/v1/models", Retryable: true}) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + ids := extractModelIDs(resp.Body) + cc.assert("non_empty", len(ids) > 0, fmt.Sprintf("models=%v", ids)) + return nil +} + +func (r *Runner) caseAdminLoginVerify(ctx context.Context, cc *caseContext) error { + loginResp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/admin/login", + Body: map[string]any{"admin_key": r.adminKey, "expire_hours": 24}, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("login_status_200", loginResp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", loginResp.StatusCode)) + var payload map[string]any + _ = json.Unmarshal(loginResp.Body, &payload) + token := asString(payload["token"]) + cc.assert("token_exists", token != "", fmt.Sprintf("body=%s", string(loginResp.Body))) + if token == "" { + return nil + } + verifyResp, err := cc.request(ctx, requestSpec{ + Method: http.MethodGet, + Path: "/admin/verify", + Headers: map[string]string{ + "Authorization": "Bearer " + token, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("verify_status_200", verifyResp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", verifyResp.StatusCode)) + var v map[string]any + _ = json.Unmarshal(verifyResp.Body, &v) + valid, _ := v["valid"].(bool) + cc.assert("verify_valid_true", valid, fmt.Sprintf("body=%s", string(verifyResp.Body))) + return nil +} + +func (r *Runner) caseAdminQueueStatus(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodGet, + Path: "/admin/queue/status", + Headers: map[string]string{ + "Authorization": "Bearer " + r.adminJWT, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + _, hasRec := m["recommended_concurrency"] + _, hasQueue := m["max_queue_size"] + cc.assert("has_recommended_concurrency", hasRec, fmt.Sprintf("body=%s", string(resp.Body))) + cc.assert("has_max_queue_size", hasQueue, fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseChatNonstream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + }, + Body: map[string]any{ + "model": "deepseek-chat", + "messages": []map[string]any{ + {"role": "user", "content": "请简单回复一句话"}, + }, + "stream": false, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + cc.assert("object_chat_completion", asString(m["object"]) == "chat.completion", fmt.Sprintf("body=%s", string(resp.Body))) + choices, _ := m["choices"].([]any) + cc.assert("choices_non_empty", len(choices) > 0, fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseChatStream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + }, + Body: map[string]any{ + "model": "deepseek-chat", + "messages": []map[string]any{ + {"role": "user", "content": "请流式回复一句话"}, + }, + "stream": true, + }, + Stream: true, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + frames, done := parseSSEFrames(resp.Body) + cc.assert("frames_non_empty", len(frames) > 0, fmt.Sprintf("len=%d", len(frames))) + cc.assert("done_terminated", done, "expected [DONE]") + return nil +} + +func (r *Runner) caseReasonerStream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + }, + Body: map[string]any{ + "model": "deepseek-reasoner", + "messages": []map[string]any{ + {"role": "user", "content": "先思考后回答：1+1"}, + }, + "stream": true, + }, + Stream: true, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + frames, done := parseSSEFrames(resp.Body) + hasReasoning := false + for _, f := range frames { + choices, _ := f["choices"].([]any) + for _, c := range choices { + ch, _ := c.(map[string]any) + delta, _ := ch["delta"].(map[string]any) + if asString(delta["reasoning_content"]) != "" { + hasReasoning = true + } + } + } + cc.assert("has_reasoning_content", hasReasoning, "reasoning_content not found") + cc.assert("done_terminated", done, "expected [DONE]") + return nil +} + +func (r *Runner) caseToolcallNonstream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + }, + Body: toolcallPayload(false), + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + choices, _ := m["choices"].([]any) + if len(choices) == 0 { + cc.assert("choices_non_empty", false, fmt.Sprintf("body=%s", string(resp.Body))) + return nil + } + c0, _ := choices[0].(map[string]any) + cc.assert("finish_reason_tool_calls", asString(c0["finish_reason"]) == "tool_calls", fmt.Sprintf("body=%s", string(resp.Body))) + msg, _ := c0["message"].(map[string]any) + tc, _ := msg["tool_calls"].([]any) + cc.assert("tool_calls_present", len(tc) > 0, fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseToolcallStream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + }, + Body: toolcallPayload(true), + Stream: true, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + frames, done := parseSSEFrames(resp.Body) + hasTool := false + rawLeak := false + for _, f := range frames { + choices, _ := f["choices"].([]any) + for _, c := range choices { + ch, _ := c.(map[string]any) + delta, _ := ch["delta"].(map[string]any) + if _, ok := delta["tool_calls"]; ok { + hasTool = true + } + content := asString(delta["content"]) + if strings.Contains(strings.ToLower(content), `"tool_calls"`) { + rawLeak = true + } + } + } + cc.assert("tool_calls_delta_present", hasTool, "tool_calls delta missing") + cc.assert("no_raw_tool_json_leak", !rawLeak, "raw tool_calls JSON leaked in content") + cc.assert("done_terminated", done, "expected [DONE]") + return nil +} + +func (r *Runner) caseAnthropicNonstream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/anthropic/v1/messages", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + Body: map[string]any{ + "model": "claude-sonnet-4-20250514", + "messages": []map[string]any{ + {"role": "user", "content": "hello"}, + }, + "stream": false, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + cc.assert("type_message", asString(m["type"]) == "message", fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseAnthropicStream(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/anthropic/v1/messages", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + Body: map[string]any{ + "model": "claude-sonnet-4-20250514", + "messages": []map[string]any{ + {"role": "user", "content": "stream hello"}, + }, + "stream": true, + }, + Stream: true, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + events := parseClaudeStreamEvents(resp.Body) + cc.assert("has_message_start", contains(events, "message_start"), fmt.Sprintf("events=%v", events)) + cc.assert("has_message_stop", contains(events, "message_stop"), fmt.Sprintf("events=%v", events)) + return nil +} + +func (r *Runner) caseAnthropicCountTokens(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/anthropic/v1/messages/count_tokens", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + Body: map[string]any{ + "model": "claude-sonnet-4-20250514", + "messages": []map[string]any{ + {"role": "user", "content": "count me"}, + }, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + v := toInt(m["input_tokens"]) + cc.assert("input_tokens_gt_zero", v > 0, fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseAdminAccountTest(ctx context.Context, cc *caseContext) error { + if strings.TrimSpace(r.accountID) == "" { + cc.assert("account_present", false, "no account in config") + return nil + } + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/admin/accounts/test", + Headers: map[string]string{ + "Authorization": "Bearer " + r.adminJWT, + }, + Body: map[string]any{ + "identifier": r.accountID, + "model": "deepseek-chat", + "message": "ping", + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_200", resp.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + ok, _ := m["success"].(bool) + cc.assert("success_true", ok, fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) caseConcurrencyBurst(ctx context.Context, cc *caseContext) error { + accountCount := len(r.configRaw.Accounts) + n := accountCount*2 + 2 + if n < 2 { + n = 2 + } + type one struct { + Status int + Err string + } + results := make([]one, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer " + r.apiKey, + }, + Body: map[string]any{ + "model": "deepseek-chat", + "messages": []map[string]any{ + {"role": "user", "content": fmt.Sprintf("并发请求 #%d，请回复ok", idx)}, + }, + "stream": true, + }, + Stream: true, + Retryable: true, + }) + if err != nil { + results[idx] = one{Err: err.Error()} + return + } + results[idx] = one{Status: resp.StatusCode} + }(i) + } + wg.Wait() + + dist := map[int]int{} + success := 0 + for _, it := range results { + if it.Status > 0 { + dist[it.Status]++ + if it.Status == http.StatusOK { + success++ + } + } + } + cc.assert("success_gt_zero", success > 0, fmt.Sprintf("distribution=%v", dist)) + _, has5xx := has5xx(dist) + cc.assert("no_5xx", !has5xx, fmt.Sprintf("distribution=%v", dist)) + if err := r.ping("/healthz"); err != nil { + cc.assert("server_alive", false, err.Error()) + } else { + cc.assert("server_alive", true, "") + } + return nil +} + +func (r *Runner) caseConfigWriteIsolated(ctx context.Context, cc *caseContext) error { + k := "testsuite-temp-" + sanitizeID(r.runID) + add, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/admin/keys", + Headers: map[string]string{ + "Authorization": "Bearer " + r.adminJWT, + }, + Body: map[string]any{"key": k}, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("add_key_status_200", add.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", add.StatusCode)) + + cfg1, err := cc.request(ctx, requestSpec{ + Method: http.MethodGet, + Path: "/admin/config", + Headers: map[string]string{ + "Authorization": "Bearer " + r.adminJWT, + }, + Retryable: true, + }) + if err != nil { + return err + } + containsAdded := strings.Contains(string(cfg1.Body), k) + cc.assert("key_present_in_isolated_config", containsAdded, "added key not found in isolated config") + + delPath := "/admin/keys/" + url.PathEscape(k) + del, err := cc.request(ctx, requestSpec{ + Method: http.MethodDelete, + Path: delPath, + Headers: map[string]string{ + "Authorization": "Bearer " + r.adminJWT, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("delete_key_status_200", del.StatusCode == http.StatusOK, fmt.Sprintf("status=%d", del.StatusCode)) + + cfg2, err := cc.request(ctx, requestSpec{ + Method: http.MethodGet, + Path: "/admin/config", + Headers: map[string]string{ + "Authorization": "Bearer " + r.adminJWT, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("key_removed_in_isolated_config", !strings.Contains(string(cfg2.Body), k), "temporary key still present") + + if err := r.ensureOriginalConfigUntouched(); err != nil { + cc.assert("original_config_unchanged", false, err.Error()) + } else { + cc.assert("original_config_unchanged", true, "") + } + return nil +} + +func (r *Runner) caseInvalidKey(ctx context.Context, cc *caseContext) error { + resp, err := cc.request(ctx, requestSpec{ + Method: http.MethodPost, + Path: "/v1/chat/completions", + Headers: map[string]string{ + "Authorization": "Bearer invalid-testsuite-key-" + sanitizeID(r.runID), + }, + Body: map[string]any{ + "model": "deepseek-chat", + "messages": []map[string]any{ + {"role": "user", "content": "hi"}, + }, + "stream": false, + }, + Retryable: true, + }) + if err != nil { + return err + } + cc.assert("status_401", resp.StatusCode == http.StatusUnauthorized, fmt.Sprintf("status=%d", resp.StatusCode)) + var m map[string]any + _ = json.Unmarshal(resp.Body, &m) + e, _ := m["error"].(map[string]any) + cc.assert("error_object_present", len(e) > 0, fmt.Sprintf("body=%s", string(resp.Body))) + cc.assert("error_message_present", asString(e["message"]) != "", fmt.Sprintf("body=%s", string(resp.Body))) + return nil +} + +func (r *Runner) doSimpleJSON(ctx context.Context, method, path string, headers map[string]string, body any) (*responseResult, error) { + cc := &caseContext{ + runner: r, + id: "auth_prepare", + traceIDsSet: map[string]struct{}{}, + } + return cc.request(ctx, requestSpec{ + Method: method, + Path: path, + Headers: headers, + Body: body, + Retryable: true, + }) +} + +func (r *Runner) writeSummary(start, end time.Time) error { + passed := 0 + failed := 0 + for _, cs := range r.results { + if cs.Passed { + passed++ + } else { + failed++ + } + } + summary := runSummary{ + RunID: r.runID, + StartedAt: start.Format(time.RFC3339Nano), + EndedAt: end.Format(time.RFC3339Nano), + DurationMS: end.Sub(start).Milliseconds(), + Stats: map[string]any{ + "total": len(r.results), + "passed": passed, + "failed": failed, + }, + Environment: map[string]any{ + "go_version": runtime.Version(), + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "base_url": r.baseURL, + "config_source": r.originalConfigPath, + "config_isolated": r.configCopyPath, + "server_log": r.serverLog, + "preflight_log": r.preflightLog, + "retries": r.opts.Retries, + "timeout_seconds": int(r.opts.Timeout.Seconds()), + }, + Cases: r.results, + Warnings: r.warnings, + } + if err := writeJSONFile(filepath.Join(r.runDir, "summary.json"), summary); err != nil { + return err + } + return os.WriteFile(filepath.Join(r.runDir, "summary.md"), []byte(r.summaryMarkdown(summary)), 0o644) +} + +func (r *Runner) summaryMarkdown(s runSummary) string { + var b strings.Builder + b.WriteString("# DS2API Live Testsuite Summary\n\n") + b.WriteString("**Sensitive Notice:** this run stores full raw request/response logs. Do not share artifacts publicly.\n\n") + fmt.Fprintf(&b, "- Run ID: `%s`\n", s.RunID) + fmt.Fprintf(&b, "- Started: `%s`\n", s.StartedAt) + fmt.Fprintf(&b, "- Ended: `%s`\n", s.EndedAt) + fmt.Fprintf(&b, "- Duration: `%d ms`\n", s.DurationMS) + fmt.Fprintf(&b, "- Passed/Failed: `%d/%d`\n\n", s.Stats["passed"], s.Stats["failed"]) + if len(s.Warnings) > 0 { + b.WriteString("## Warnings\n\n") + for _, w := range s.Warnings { + fmt.Fprintf(&b, "- %s\n", w) + } + b.WriteString("\n") + } + b.WriteString("## Failed Cases\n\n") + hasFailed := false + for _, c := range s.Cases { + if c.Passed { + continue + } + hasFailed = true + fmt.Fprintf(&b, "- `%s`: %s\n", c.CaseID, c.Error) + if len(c.TraceIDs) > 0 { + fmt.Fprintf(&b, " - trace_ids: `%s`\n", strings.Join(c.TraceIDs, ", ")) + fmt.Fprintf(&b, " - grep: `rg \"%s\" %s`\n", c.TraceIDs[0], filepath.Join(r.runDir, "server.log")) + } + fmt.Fprintf(&b, " - artifact: `%s`\n", c.ArtifactPath) + } + if !hasFailed { + b.WriteString("- none\n") + } + b.WriteString("\n## Case Table\n\n") + b.WriteString("| case_id | status | duration_ms | statuses | artifact |\n") + b.WriteString("|---|---:|---:|---|---|\n") + for _, c := range s.Cases { + status := "PASS" + if !c.Passed { + status = "FAIL" + } + fmt.Fprintf(&b, "| %s | %s | %d | %v | `%s` |\n", c.CaseID, status, c.DurationMS, c.StatusCodes, c.ArtifactPath) + } + return b.String() +} + +func toolcallPayload(stream bool) map[string]any { + return map[string]any{ + "model": "deepseek-chat", + "messages": []map[string]any{ + { + "role": "user", + "content": "你必须调用工具 search 查询 golang，并仅返回工具调用。", + }, + }, + "tools": []map[string]any{ + { + "type": "function", + "function": map[string]any{ + "name": "search", + "description": "search documents", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{ + "q": map[string]any{ + "type": "string", + }, + }, + "required": []string{"q"}, + }, + }, + }, + }, + "stream": stream, + } +} + +func parseSSEFrames(body []byte) ([]map[string]any, bool) { + lines := strings.Split(string(body), "\n") + frames := make([]map[string]any, 0, len(lines)) + done := false + for _, line := range lines { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "" { + continue + } + if payload == "[DONE]" { + done = true + continue + } + var m map[string]any + if err := json.Unmarshal([]byte(payload), &m); err == nil { + frames = append(frames, m) + } + } + return frames, done +} + +func parseClaudeStreamEvents(body []byte) []string { + events := []string{} + seen := map[string]bool{} + lines := strings.Split(string(body), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "" { + continue + } + var m map[string]any + if err := json.Unmarshal([]byte(payload), &m); err != nil { + continue + } + t := asString(m["type"]) + if t == "" || seen[t] { + continue + } + seen[t] = true + events = append(events, t) + } + return events +} + +func extractModelIDs(body []byte) []string { + var m map[string]any + if err := json.Unmarshal(body, &m); err != nil { + return nil + } + out := []string{} + data, _ := m["data"].([]any) + for _, it := range data { + item, _ := it.(map[string]any) + id := asString(item["id"]) + if id != "" { + out = append(out, id) + } + } + return out +} + +func withTraceQuery(rawURL, traceID string) (string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", err + } + q := u.Query() + q.Set("__trace_id", traceID) + u.RawQuery = q.Encode() + return u.String(), nil +} + +func writeJSONFile(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, b, 0o644) +} + +func prepareServerEnv(base []string, overrides map[string]string) []string { + out := make([]string, 0, len(base)+len(overrides)) + skip := map[string]struct{}{} + for k := range overrides { + skip[k] = struct{}{} + } + for _, e := range base { + parts := strings.SplitN(e, "=", 2) + if len(parts) != 2 { + continue + } + if _, ok := skip[parts[0]]; ok { + continue + } + out = append(out, e) + } + for k, v := range overrides { + out = append(out, k+"="+v) + } + return out +} + +func findFreePort() (int, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer ln.Close() + addr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + return 0, errors.New("failed to detect tcp port") + } + return addr.Port, nil +} + +func uniqueStatusCodes(in []responseLog) []int { + set := map[int]struct{}{} + for _, it := range in { + if it.StatusCode > 0 { + set[it.StatusCode] = struct{}{} + } + } + out := make([]int, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Ints(out) + return out +} + +func has5xx(dist map[int]int) (int, bool) { + for k := range dist { + if k >= 500 { + return k, true + } + } + return 0, false +} + +func sanitizeID(s string) string { + s = strings.ReplaceAll(s, ":", "_") + s = strings.ReplaceAll(s, "/", "_") + s = strings.ReplaceAll(s, " ", "_") + return s +} + +func asString(v any) string { + if v == nil { + return "" + } + switch x := v.(type) { + case string: + return strings.TrimSpace(x) + default: + return strings.TrimSpace(fmt.Sprintf("%v", v)) + } +} + +func toInt(v any) int { + switch x := v.(type) { + case float64: + return int(x) + case float32: + return int(x) + case int: + return x + case int64: + return int(x) + default: + return 0 + } +} + +func contains(xs []string, target string) bool { + for _, x := range xs { + if x == target { + return true + } + } + return false +} diff --git a/scripts/testsuite/run-live.sh b/scripts/testsuite/run-live.sh new file mode 100755 index 0000000..eeefdc2 --- /dev/null +++ b/scripts/testsuite/run-live.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT_DIR" + +go run ./cmd/ds2api-tests "$@" +