diff --git a/API.en.md b/API.en.md index 07fbf3d..16f3f99 100644 --- a/API.en.md +++ b/API.en.md @@ -40,7 +40,7 @@ Docs: [Overview](README.en.md) / [Architecture](docs/ARCHITECTURE.en.md) / [Depl - OpenAI / Claude / Gemini protocols are now mounted on one shared `chi` router tree assembled in `internal/server/router.go`. - Adapter responsibilities are streamlined to: **request normalization → DeepSeek invocation → protocol-shaped rendering**, reducing legacy split-logic paths. -- Tool-calling semantics are aligned between Go and Node runtime: models should output the fullwidth-separator DSML shell `<｜DSML｜tool_calls>` → `<｜DSML｜invoke name="...">` → `<｜DSML｜parameter name="...">`; DS2API also accepts the halfwidth DSML wrapper `<|DSML|tool_calls>`, DSML wrapper aliases such as ``, `<|tool_calls>`, `<｜tool_calls>`, common DSML separator drift such as `<|DSML tool_calls>`, collapsed DSML local names such as ``, control-separator drift such as `` / raw STX `\x02`, CJK angle bracket, fullwidth-bang / ideographic-comma separator drift, and trailing attribute separator drift such as `...〈/DSM｜parameter〉`, `<！DSML！invoke name=“Bash”>`, or `<、DSML、tool_calls>`, arbitrary protocol prefixes such as ``, and legacy canonical XML `` → `` → ``. The scanner normalizes fixed local names (`tool_calls` / `invoke` / `parameter`) back to XML before parsing, and also tolerates CDATA opener drift such as `<！[CDATA[` / `<、[CDATA[`; only wrapped tool blocks or the narrow missing-opening-wrapper repair path enter the tool path, while bare `` does not count as supported syntax. JSON literal parameter bodies are preserved as structured values, explicit empty or whitespace-only parameters are preserved as empty strings, malformed complete wrappers are released as plain text, and loose CDATA is narrowly repaired at final parse/flush when it can preserve a complete outer tool call. +- Tool-calling semantics are aligned between Go and Node runtime: models should output the fullwidth-separator DSML shell `<｜DSML｜tool_calls>` → `<｜DSML｜invoke name="...">` → `<｜DSML｜parameter name="...">`; DS2API also accepts the halfwidth DSML wrapper `<|DSML|tool_calls>`, DSML wrapper aliases such as ``, `<|tool_calls>`, `<｜tool_calls>`, common DSML separator drift such as `<|DSML tool_calls>`, collapsed DSML local names such as ``, control-separator drift such as `` / raw STX `\x02`, CJK angle bracket, fullwidth-bang / ideographic-comma separator drift, and trailing attribute separator drift such as `...〈/DSM｜parameter〉`, `<！DSML！invoke name=“Bash”>`, `<、DSML、tool_calls>`, or ``, arbitrary protocol prefixes such as ``, and legacy canonical XML `` → `` → ``. The scanner normalizes fixed local names (`tool_calls` / `invoke` / `parameter`) with non-structural separators before or after them back to XML before parsing, and also tolerates CDATA opener drift such as `<！[CDATA[` / `<、[CDATA[`; only wrapped tool blocks or the narrow missing-opening-wrapper repair path enter the tool path, while bare `` does not count as supported syntax. JSON literal parameter bodies are preserved as structured values, explicit empty or whitespace-only parameters are preserved as empty strings, malformed complete wrappers are released as plain text, and loose CDATA is narrowly repaired at final parse/flush when it can preserve a complete outer tool call. - `Admin API` separates static config from runtime policy: `/admin/config*` for configuration state, `/admin/settings*` for runtime behavior. - When upstream returns a thinking-only response with no visible text, the Go main path for both streaming and non-streaming completions retries once in the same DeepSeek session: it appends the prompt suffix `"Previous reply had no visible output. Please regenerate the visible final answer or tool call now."` and sets `parent_message_id`. If that same-account retry would still end as `429 upstream_empty_output`, managed-account mode switches to the next available account, creates a fresh session, and retries the original payload once before returning 429. - Citation/reference marker boundary: streaming output hides upstream `[citation:N]` / `[reference:N]` placeholders by default; non-stream output converts DeepSeek search reference markers into Markdown links. @@ -355,7 +355,7 @@ When `tools` is present, DS2API performs anti-leak handling: Additional notes: -- The parser treats the recommended DSML shell tool blocks (`<｜DSML｜tool_calls>` / `<｜DSML｜invoke name="...">` / `<｜DSML｜parameter name="...">`), halfwidth DSML shell blocks (`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`), DSML wrapper aliases (``, `<|tool_calls>`, `<｜tool_calls>`), common DSML separator drift (`<|DSML tool_calls>` / `<|DSML invoke>` / `<|DSML parameter>`), collapsed DSML local names (`` / `` / ``), control-separator drift (`` / raw STX `\x02`), CJK angle bracket, fullwidth-bang / ideographic-comma separator drift, and trailing attribute separator drift (`...〈/DSM｜parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>`), arbitrary protocol prefixes (``), and legacy canonical XML tool blocks (`` / `` / ``) as executable tool calls. These shells normalize back to XML first, while internal parsing remains XML-based; CDATA opener drift such as `<！[CDATA[` / `<、[CDATA[` is also normalized for parameter bodies. Legacy ``, ``, ``, ``, ``, `tool_use`, antml variants, and standalone JSON `tool_calls` payloads are treated as plain text; complete but malformed wrappers are also released as plain text. +- The parser treats the recommended DSML shell tool blocks (`<｜DSML｜tool_calls>` / `<｜DSML｜invoke name="...">` / `<｜DSML｜parameter name="...">`), halfwidth DSML shell blocks (`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`), DSML wrapper aliases (``, `<|tool_calls>`, `<｜tool_calls>`), common DSML separator drift (`<|DSML tool_calls>` / `<|DSML invoke>` / `<|DSML parameter>`), collapsed DSML local names (`` / `` / ``), control-separator drift (`` / raw STX `\x02`), CJK angle bracket, fullwidth-bang / ideographic-comma separator drift, and trailing attribute separator drift (`...〈/DSM｜parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>` / ``), arbitrary protocol prefixes (``), and legacy canonical XML tool blocks (`` / `` / ``) as executable tool calls. These shells normalize non-structural separators back to XML first, while internal parsing remains XML-based; CDATA opener drift such as `<！[CDATA[` / `<、[CDATA[` is also normalized for parameter bodies. Legacy ``, ``, ``, ``, ``, `tool_use`, antml variants, and standalone JSON `tool_calls` payloads are treated as plain text; complete but malformed wrappers are also released as plain text. - The parser no longer drops tool calls solely because parameter values are empty; explicit empty strings or whitespace-only parameters become empty strings in structured `tool_calls`. Prompting still tells the model not to emit blank parameters, and missing/empty argument rejection belongs in the tool executor or client schema validation. - If the final visible response text is empty but the reasoning stream contains an executable tool call, Chat / Responses emits a standard OpenAI `tool_calls` / `function_call` output during finalization. If thinking/reasoning was not enabled by the client, that reasoning text is used only for detection and is not exposed as visible text or `reasoning_content`. - `tool_calls` shown inside fenced markdown code blocks (for example, ```json ... ```) are treated as examples, not executable calls. diff --git a/API.md b/API.md index 5ad14c0..c39c496 100644 --- a/API.md +++ b/API.md @@ -40,7 +40,7 @@ - OpenAI / Claude / Gemini 三套协议已统一挂在同一 `chi` 路由树上，由 `internal/server/router.go` 负责装配。 - 适配器层职责收敛为：**请求归一化 → DeepSeek 调用 → 协议形态渲染**，减少历史版本中“同能力多处实现”的分叉。 -- Tool Calling 的解析策略在 Go 与 Node Runtime 间保持一致：推荐模型输出全角分隔符 DSML 外壳 `<｜DSML｜tool_calls>` → `<｜DSML｜invoke name="...">` → `<｜DSML｜parameter name="...">`；兼容层也接受半角 DSML wrapper `<|DSML|tool_calls>`、DSML wrapper 别名 ``、`<|tool_calls>`、`<｜tool_calls>`、常见 DSML 分隔符漏写形态（如 `<|DSML tool_calls>`）、`DSML` 与工具标签名黏连的常见 typo（如 ``）、控制分隔符漂移（如 `` / 原始 STX `\x02`）、CJK 尖括号、全角感叹号、顿号、弯引号属性值与属性尾部分隔符漂移（如 `...〈/DSM｜parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>`）、任意协议前缀壳（如 ``），以及旧式 canonical XML `` → `` → ``。实现上采用结构扫描：只要固定本地标签名是 `tool_calls` / `invoke` / `parameter`，前缀壳会在解析入口归一化；CDATA 开头也会容错 `<！[CDATA[` / `<、[CDATA[` 这类分隔符漂移；只有 `tool_calls` wrapper 或可修复的缺失 opening wrapper 会进入工具路径，裸 `` 不计为已支持语法；流式场景继续执行防泄漏筛分。若参数体本身是合法 JSON 字面量（如 `123`、`true`、`null`、数组或对象），会按结构化值输出，不再一律当作字符串；显式空字符串和纯空白参数会结构化保留为空字符串，是否拒绝缺参由工具执行侧决定；完整但 malformed 的 wrapper 会作为普通文本释放，不会吞掉或伪造成工具调用；若 CDATA 偶发漏闭合，则会在最终 parse / flush 恢复阶段做窄修复，尽量保住已完整包裹的外层工具调用。 +- Tool Calling 的解析策略在 Go 与 Node Runtime 间保持一致：推荐模型输出全角分隔符 DSML 外壳 `<｜DSML｜tool_calls>` → `<｜DSML｜invoke name="...">` → `<｜DSML｜parameter name="...">`；兼容层也接受半角 DSML wrapper `<|DSML|tool_calls>`、DSML wrapper 别名 ``、`<|tool_calls>`、`<｜tool_calls>`、常见 DSML 分隔符漏写形态（如 `<|DSML tool_calls>`）、`DSML` 与工具标签名黏连的常见 typo（如 ``）、控制分隔符漂移（如 `` / 原始 STX `\x02`）、CJK 尖括号、全角感叹号、顿号、弯引号属性值与属性尾部分隔符漂移（如 `...〈/DSM｜parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>` / ``）、任意协议前缀壳（如 ``），以及旧式 canonical XML `` → `` → ``。实现上采用结构扫描：只要固定本地标签名是 `tool_calls` / `invoke` / `parameter`，标签名前或标签名后的非结构性分隔符会在解析入口归一化；CDATA 开头也会容错 `<！[CDATA[` / `<、[CDATA[` 这类分隔符漂移；只有 `tool_calls` wrapper 或可修复的缺失 opening wrapper 会进入工具路径，裸 `` 不计为已支持语法；流式场景继续执行防泄漏筛分。若参数体本身是合法 JSON 字面量（如 `123`、`true`、`null`、数组或对象），会按结构化值输出，不再一律当作字符串；显式空字符串和纯空白参数会结构化保留为空字符串，是否拒绝缺参由工具执行侧决定；完整但 malformed 的 wrapper 会作为普通文本释放，不会吞掉或伪造成工具调用；若 CDATA 偶发漏闭合，则会在最终 parse / flush 恢复阶段做窄修复，尽量保住已完整包裹的外层工具调用。 - `Admin API` 将配置与运行时策略分开：`/admin/config*` 管静态配置，`/admin/settings*` 管运行时行为。 - 当上游返回 thinking-only 响应（模型输出了推理链但无可见文本）时，Go 主路径的流式与非流式补全都会先自动重试一次：以多轮对话 follow-up 方式追加 prompt 后缀 `"Previous reply had no visible output. Please regenerate the visible final answer or tool call now."` 并设置 `parent_message_id` 在同一 DeepSeek session 内让模型重新输出；同账号重试最大 1 次。若同账号重试后仍即将返回 `429 upstream_empty_output`，托管账号模式会在返回 429 前自动切换到下一个可用账号，新建 session，用原始 payload 再 fresh retry 一次。 - 引用标记处理边界：流式输出默认隐藏 `[citation:N]` / `[reference:N]` 这类上游内部占位符；非流式输出默认把 DeepSeek 搜索引用标记转换为 Markdown 引用链接。 @@ -357,7 +357,7 @@ data: [DONE] 补充说明： - **非代码块上下文**下，工具负载即使与普通文本混合，也会按特征识别并产出可执行 tool call（前后普通文本仍可透传）。 -- 解析器当前把推荐 DSML 外壳（`<｜DSML｜tool_calls>` / `<｜DSML｜invoke name="...">` / `<｜DSML｜parameter name="...">`）、半角 DSML 外壳（`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`）、DSML wrapper 别名（``、`<|tool_calls>`、`<｜tool_calls>`）、常见 DSML 分隔符漏写形态（如 `<|DSML tool_calls>` / `<|DSML invoke>` / `<|DSML parameter>`）、`DSML` 与工具标签名黏连的常见 typo（如 `` / `` / ``）、控制分隔符漂移（如 `` / 原始 STX `\x02`）、CJK 尖括号、全角感叹号、顿号、弯引号属性值与属性尾部分隔符漂移（如 `...〈/DSM｜parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>`）、任意协议前缀壳（如 ``）和旧式 canonical XML 工具块（`` / `` / ``）作为可执行调用解析；这些前缀壳会先归一化回 XML，内部仍以 XML 解析语义为准，CDATA 开头也会容错 `<！[CDATA[` / `<、[CDATA[`。旧式 ``、``、``、``、``、`tool_use`、antml 风格与纯 JSON `tool_calls` 片段默认都会按普通文本处理；完整但 malformed 的 wrapper 同样会作为普通文本释放。 +- 解析器当前把推荐 DSML 外壳（`<｜DSML｜tool_calls>` / `<｜DSML｜invoke name="...">` / `<｜DSML｜parameter name="...">`）、半角 DSML 外壳（`<|DSML|tool_calls>` / `<|DSML|invoke name="...">` / `<|DSML|parameter name="...">`）、DSML wrapper 别名（``、`<|tool_calls>`、`<｜tool_calls>`）、常见 DSML 分隔符漏写形态（如 `<|DSML tool_calls>` / `<|DSML invoke>` / `<|DSML parameter>`）、`DSML` 与工具标签名黏连的常见 typo（如 `` / `` / ``）、控制分隔符漂移（如 `` / 原始 STX `\x02`）、CJK 尖括号、全角感叹号、顿号、弯引号属性值与属性尾部分隔符漂移（如 `...〈/DSM｜parameter〉` / `<！DSML！invoke name=“Bash”>` / `<、DSML、tool_calls>` / ``）、任意协议前缀壳（如 ``）和旧式 canonical XML 工具块（`` / `` / ``）作为可执行调用解析；这些非结构性分隔符壳会先归一化回 XML，内部仍以 XML 解析语义为准，CDATA 开头也会容错 `<！[CDATA[` / `<、[CDATA[`。旧式 ``、``、``、``、``、`tool_use`、antml 风格与纯 JSON `tool_calls` 片段默认都会按普通文本处理；完整但 malformed 的 wrapper 同样会作为普通文本释放。 - 解析层不会因为参数值为空而丢弃工具调用；显式空字符串或纯空白参数会按空字符串进入结构化 `tool_calls`。Prompt 会要求模型不要主动输出空参数，缺参/空命令的拒绝应由工具执行侧或客户端 schema 校验负责。 - 当最终可见正文为空但思维链里包含可执行工具调用时，Chat / Responses 会在收尾阶段补发标准 OpenAI `tool_calls` / `function_call` 输出；如果客户端未开启 thinking / reasoning，该思维链只用于检测，不会作为可见正文或 `reasoning_content` 暴露。 - Markdown fenced code block（例如 ```json ... ```）中的 `tool_calls` 仅视为示例文本，不会被执行。 diff --git a/docs/prompt-compatibility.md b/docs/prompt-compatibility.md index 9d2494a..e0cb4a3 100644 --- a/docs/prompt-compatibility.md +++ b/docs/prompt-compatibility.md @@ -168,7 +168,7 @@ OpenAI Chat / Responses 在标准化后、current input file 之前，会默认 4. 把这整段内容并入 system prompt。 工具调用正例现在优先示范全角分隔符 DSML 风格：`<｜DSML｜tool_calls>` → `<｜DSML｜invoke name="...">` → `<｜DSML｜parameter name="...">`。 -兼容层仍接受旧式纯 `` wrapper，并会容错若干 DSML 标签变体，包括短横线形式 `` / `` / ``、下划线形式 `` / `` / ``，以及其他前缀分隔形态如 `` / `` / ``；标签壳扫描还会把全角 ASCII 漂移归一化，例如 `<ｄＳＭＬ｜tool_calls>` 与全角 `＞` 结束符，也会容错 CJK 尖括号、全角感叹号或顿号分隔符、弯引号属性值和属性尾部分隔符漂移，例如 `...〈/DSM｜parameter〉`、`<！DSML！invoke name=“Bash”>`、`<、DSML、tool_calls>`。更一般地，Go / Node tag 扫描以固定本地标签名 `tool_calls` / `invoke` / `parameter` 为准，标签名前任意协议前缀壳都会在解析入口剥离，例如 ``、`` 这类控制符或非 ASCII 分隔符漂移也会归一化回现有 XML 标签后继续走同一套 parser。CDATA 开头也使用同一类扫描式容错，`` wrapper，并会容错若干 DSML 标签变体，包括短横线形式 `` / `` / ``、下划线形式 `` / `` / ``，以及其他前缀分隔形态如 `` / `` / ``；标签壳扫描还会把全角 ASCII 漂移归一化，例如 `<ｄＳＭＬ｜tool_calls>` 与全角 `＞` 结束符，也会容错 CJK 尖括号、全角感叹号或顿号分隔符、弯引号属性值和属性尾部分隔符漂移，例如 `...〈/DSM｜parameter〉`、`<！DSML！invoke name=“Bash”>`、`<、DSML、tool_calls>`、``。更一般地，Go / Node tag 扫描以固定本地标签名 `tool_calls` / `invoke` / `parameter` 为准，标签名前或标签名后的非结构性协议分隔符都会在解析入口剥离，例如 ``、`` 这类控制符或非 ASCII 分隔符漂移也会归一化回现有 XML 标签后继续走同一套 parser；结构性字符如 `<` / `>` / `/` / `=` / 引号、空白和 ASCII 字母数字不会被当作这类分隔符。CDATA 开头也使用同一类扫描式容错，`...` 子节点表示；当某个参数体只包含 item 子节点时，Go / Node 解析器会把它还原成数组，避免 `questions` / `options` 这类 schema 中要求 array 的参数被误解析成 `{ "item": ... }` 对象。除此之外，解析器还会回收一些更松散的列表写法，例如 JSON array 字面量或逗号分隔的 JSON 项序列，只要它们足够明确；但 `` 仍然是首选形态。若模型把完整结构化 XML fragment 误包进 CDATA，兼容层会在保护 `content` / `command` 等原文字段的前提下，尝试把非原文字段中的 CDATA XML fragment 还原成 object / array。不过，如果 CDATA 只是单个平面的 XML/HTML 标签，例如 `urgent` 这种行内标记，兼容层会保留原始字符串，不会强行升成 object / array；只有明显表示结构的 CDATA 片段，例如多兄弟节点、嵌套子节点或 `item` 列表，才会触发结构化恢复。对 `command` / `content` 等长文本参数，CDATA 内部的 Markdown fenced DSML / XML 示例会作为原文保护；示例里的 `]]>` 或 `` 不会截断外层工具调用，解析器会继续等待围栏外真正的参数 / wrapper 结束标签。 Go 侧读取 DeepSeek SSE 时不再依赖 `bufio.Scanner` 的固定 2MiB 单行上限；当写文件类工具把很长的 `content` 放在单个 `data:` 行里返回时，非流式收集、流式解析和 auto-continue 透传都会保留完整行，再进入同一套工具解析与序列化流程。 在 assistant 最终回包阶段，如果某个 tool 参数在声明 schema 中明确是 `string`，兼容层会在把解析后的 `tool_calls` / `function_call` 重新序列化成 OpenAI / Responses / Claude 可见参数前，递归把该路径上的 number / bool / object / array 统一转成字符串；其中 object / array 会压成紧凑 JSON 字符串。这个保护只对 schema 明确声明为 string 的路径生效，不会改写本来就是 `number` / `boolean` / `object` / `array` 的参数。这样可以兼容 DeepSeek 输出了结构化片段、但上游客户端工具 schema 又严格要求字符串参数的场景（例如 `content`、`prompt`、`path`、`taskId` 等）。 diff --git a/docs/toolcall-semantics.md b/docs/toolcall-semantics.md index 409b942..23358e2 100644 --- a/docs/toolcall-semantics.md +++ b/docs/toolcall-semantics.md @@ -40,7 +40,7 @@ - 如果模型漏掉 opening wrapper，但后面仍输出了一个或多个 invoke 并以 closing wrapper 收尾，Go 解析链路会在解析前补回缺失的 opening wrapper。 - Go / Node 解析层不再枚举每一种 DSML typo。它以固定本地标签名 `tool_calls` / `invoke` / `parameter` 为准，把标签名前的任意协议前缀壳视为可容忍噪声，并继续兼容管道符 `|` / `｜`、全角感叹号 `！`、顿号 `、`、空白、重复 leading `<`、可视控制符 `␂`、原始 STX `\x02`、非 ASCII 分隔符、CJK 尖括号 `〈` / `〉`、弯引号属性值等漂移。例如 ``、`<<|DSML|tool_calls>`、`<|DSML tool_calls>`、``、`<`、``、``、`...〈/DSM｜tool_calls〉`、`<！DSML！tool_calls>...<！/DSML！tool_calls>`、`<、DSML、tool_calls>...<、/DSML、tool_calls>` 都会归一化；相似但非固定标签名（如 `tool_calls_extra`）仍按普通文本处理。 -- 如果模型在固定工具标签名后多输出一个尾部管道符，例如 `<|DSML|tool_calls|` / `<|DSML|invoke|` / `<|DSML|parameter|`，或在带属性标签的结束符前多输出一个尾部管道符（如 ``），兼容层会把这个尾部 `|` / `｜` 当作异常标签终止符并补齐或归一化；如果后面已经有 `>` / `〉`，也会消费这个多余分隔符后再归一化。 +- 如果模型在固定工具标签名后多输出一个非结构性分隔符，例如 `<|DSML|tool_calls|` / `<|DSML|invoke|` / `<|DSML|parameter|` / ``，或在带属性标签的结束符前多输出一个尾部分隔符（如 ``），兼容层会把这个尾部分隔符当作异常标签终止符并补齐或归一化；如果后面已经有 `>` / `〉`，也会消费这个多余分隔符后再归一化。结构性字符如 `<` / `>` / `/` / `=` / 引号、空白和 ASCII 字母数字不会被当作这类分隔符。 - 这是一个针对常见模型失误的窄修复，不改变推荐输出格式；prompt 仍要求模型直接输出完整 DSML 外壳。 - 裸 `` / `` 不会被当成“已支持的工具语法”；只有 `tool_calls` wrapper 或可修复的缺失 opening wrapper 才会进入工具调用路径。 @@ -54,7 +54,7 @@ 在流式链路中（Go / Node 一致）： -- DSML `<｜DSML｜tool_calls>` wrapper、短横线形式（如 `` / `` / ``）、基于固定本地标签名的 DSML 噪声容错形态、尾部管道符形态（如 `<|DSML|tool_calls|`）和 canonical `` wrapper 都会进入结构化捕获 +- DSML `<｜DSML｜tool_calls>` wrapper、短横线形式（如 `` / `` / ``）、基于固定本地标签名的 DSML 噪声容错形态、尾部非结构性分隔符形态（如 `<|DSML|tool_calls|` / ``）和 canonical `` wrapper 都会进入结构化捕获 - 如果流里直接从 invoke 开始，但后面补上了 closing wrapper，Go 流式筛分也会按缺失 opening wrapper 的修复路径尝试恢复 - 已识别成功的工具调用不会再次回流到普通文本 - 不符合新格式的块不会执行，并继续按原样文本透传 diff --git a/internal/js/helpers/stream-tool-sieve/parse_payload.js b/internal/js/helpers/stream-tool-sieve/parse_payload.js index 175223c..11b31a4 100644 --- a/internal/js/helpers/stream-tool-sieve/parse_payload.js +++ b/internal/js/helpers/stream-tool-sieve/parse_payload.js @@ -464,14 +464,7 @@ function matchCDATAOpenAt(text, start) { } function isCDATAOpenSeparator(ch) { - const normalized = normalizeFullwidthASCIIChar(ch || ''); - if (!normalized || ['<', '>', '/', '=', '"', "'", '['].includes(normalized)) { - return false; - } - if ([' ', '\t', '\n', '\r'].includes(normalized)) { - return false; - } - return !/^[A-Za-z0-9]$/.test(normalized); + return isToolMarkupSeparator(ch); } function findCDATAEnd(text, from) { @@ -528,21 +521,21 @@ function scanToolMarkupTagAt(text, start) { } const originalNameEnd = i + len; let nameEnd = originalNameEnd; - while (nameEnd < raw.length && isToolMarkupPipe(raw[nameEnd])) { + while (nameEnd < raw.length && isToolMarkupSeparator(raw[nameEnd])) { nameEnd += 1; } - const hasTrailingPipe = nameEnd > originalNameEnd; + const hasTrailingSeparator = nameEnd > originalNameEnd; if (!hasXmlTagBoundary(raw, nameEnd)) { return null; } let end = findXmlTagEnd(raw, nameEnd); if (end < 0) { - if (!hasTrailingPipe) { + if (!hasTrailingSeparator) { return null; } end = nameEnd - 1; } - if (hasTrailingPipe) { + if (hasTrailingSeparator) { const nextLT = raw.indexOf('<', nameEnd); if (nextLT >= 0 && end >= nextLT) { end = nameEnd - 1; @@ -635,8 +628,15 @@ function includeDuplicateLeadingLessThan(text, idx) { return out; } -function isToolMarkupPipe(ch) { - return ch === '|' || ch === '｜' || ch === '␂' || ch === '\x02'; +function isToolMarkupSeparator(ch) { + const normalized = normalizeFullwidthASCIIChar(ch || ''); + if (!normalized || ['<', '>', '/', '=', '"', "'", '['].includes(normalized)) { + return false; + } + if ([' ', '\t', '\n', '\r'].includes(normalized)) { + return false; + } + return !/^[A-Za-z0-9]$/.test(normalized); } function isPartialToolMarkupTagPrefix(text) { @@ -755,7 +755,7 @@ function isToolMarkupTagTerminator(raw, idx) { } function consumeToolMarkupNamePrefixOnce(raw, lower, idx) { - if (idx < raw.length && isToolMarkupPipe(raw[idx])) { + if (idx < raw.length && isToolMarkupSeparator(raw[idx])) { return { next: idx + 1, ok: true }; } if (idx < raw.length && [' ', '\t', '\r', '\n'].includes(raw[idx])) { @@ -794,7 +794,7 @@ function consumeArbitraryToolMarkupNamePrefix(raw, lower, idx) { } let next = k; let ok = false; - if (next < raw.length && isToolMarkupPipe(raw[next])) { + if (next < raw.length && isToolMarkupSeparator(raw[next])) { next += 1; ok = true; } else if (next < raw.length && ['_', '-'].includes(normalizeFullwidthASCIIChar(raw[next]))) { diff --git a/internal/toolcall/toolcalls_markup.go b/internal/toolcall/toolcalls_markup.go index cc94256..08cf07e 100644 --- a/internal/toolcall/toolcalls_markup.go +++ b/internal/toolcall/toolcalls_markup.go @@ -9,9 +9,6 @@ import ( var toolCallMarkupKVPattern = regexp.MustCompile(`(?is)<(?:[a-z0-9_:-]+:)?([a-z0-9_\-.]+)\b[^>]*>(.*?)`) -// cdataPattern matches a standalone CDATA section. -var cdataPattern = regexp.MustCompile(`(?is)^(?:<|〈)(?:!|！)\[CDATA\[(.*?)]](?:>|＞|〉)$`) - func parseMarkupKVObject(text string) map[string]any { matches := toolCallMarkupKVPattern.FindAllStringSubmatch(strings.TrimSpace(text), -1) if len(matches) == 0 { diff --git a/internal/toolcall/toolcalls_parse_markup.go b/internal/toolcall/toolcalls_parse_markup.go index 8c72f74..4660c50 100644 --- a/internal/toolcall/toolcalls_parse_markup.go +++ b/internal/toolcall/toolcalls_parse_markup.go @@ -247,17 +247,7 @@ func matchToolCDATAOpenAt(text string, start int) (int, bool) { } func isToolCDATAOpenSeparator(r rune) bool { - ch := normalizeFullwidthASCII(r) - if ch == 0 || ch == '<' || ch == '>' || ch == '/' || ch == '=' || ch == '"' || ch == '\'' || ch == '[' { - return false - } - if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' { - return false - } - if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') { - return false - } - return true + return isToolMarkupSeparator(r) } func hasASCIIPrefixFoldAt(text string, start int, prefix string) bool { diff --git a/internal/toolcall/toolcalls_scan.go b/internal/toolcall/toolcalls_scan.go index 30b4e6a..e331e43 100644 --- a/internal/toolcall/toolcalls_scan.go +++ b/internal/toolcall/toolcalls_scan.go @@ -168,22 +168,22 @@ func scanToolMarkupTagAt(text string, start int) (ToolMarkupTag, bool) { dsmlLike = true } nameEnd := i + nameLen - nameEndBeforePipes := nameEnd - for next, ok := consumeToolMarkupPipe(text, nameEnd); ok; next, ok = consumeToolMarkupPipe(text, nameEnd) { + nameEndBeforeSeparators := nameEnd + for next, ok := consumeToolMarkupSeparator(text, nameEnd); ok; next, ok = consumeToolMarkupSeparator(text, nameEnd) { nameEnd = next } - hasTrailingPipe := nameEnd > nameEndBeforePipes + hasTrailingSeparator := nameEnd > nameEndBeforeSeparators if !hasToolMarkupBoundary(text, nameEnd) { return ToolMarkupTag{}, false } end := findXMLTagEnd(text, nameEnd) if end < 0 { - if !hasTrailingPipe { + if !hasTrailingSeparator { return ToolMarkupTag{}, false } end = nameEnd - 1 } - if hasTrailingPipe { + if hasTrailingSeparator { if nextLT := strings.IndexByte(text[nameEnd:], '<'); nextLT >= 0 && end >= nameEnd+nextLT { end = nameEnd - 1 } @@ -251,7 +251,7 @@ func consumeToolMarkupNamePrefix(text string, idx int) (int, bool) { } func consumeToolMarkupNamePrefixOnce(text string, idx int) (int, bool) { - if next, ok := consumeToolMarkupPipe(text, idx); ok { + if next, ok := consumeToolMarkupSeparator(text, idx); ok { return next, true } if idx < len(text) && (text[idx] == ' ' || text[idx] == '\t' || text[idx] == '\r' || text[idx] == '\n') { @@ -288,7 +288,7 @@ func consumeArbitraryToolMarkupNamePrefix(text string, idx int) (int, bool) { for k < len(text) && (text[k] == ' ' || text[k] == '\t' || text[k] == '\r' || text[k] == '\n') { k++ } - next, ok := consumeToolMarkupPipe(text, k) + next, ok := consumeToolMarkupSeparator(text, k) if !ok { if sep, size := normalizedASCIIAt(text, k); sep == '_' || sep == '-' { next = k + size @@ -448,26 +448,29 @@ func isToolMarkupTagTerminator(text string, idx int) bool { return normalizeFullwidthASCII(r) == '>' } -func consumeToolMarkupPipe(text string, idx int) (int, bool) { +func consumeToolMarkupSeparator(text string, idx int) (int, bool) { if idx >= len(text) { return idx, false } - if text[idx] == '|' { - return idx + 1, true + r, size := utf8.DecodeRuneInString(text[idx:]) + if size <= 0 || !isToolMarkupSeparator(r) { + return idx, false } - if text[idx] == '\x02' { - return idx + 1, true + return idx + size, true +} + +func isToolMarkupSeparator(r rune) bool { + ch := normalizeFullwidthASCII(r) + if ch == 0 || ch == '<' || ch == '>' || ch == '/' || ch == '=' || ch == '"' || ch == '\'' || ch == '[' { + return false } - if strings.HasPrefix(text[idx:], "｜") { - return idx + len("｜"), true + if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' { + return false } - if strings.HasPrefix(text[idx:], "␂") { - return idx + len("␂"), true + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') { + return false } - if ch, size := normalizedASCIIAt(text, idx); ch == '!' { - return idx + size, true - } - return idx, false + return true } func consumeToolMarkupLessThan(text string, idx int) (int, bool) { diff --git a/internal/toolcall/toolcalls_test.go b/internal/toolcall/toolcalls_test.go index 1171b8d..01d7d10 100644 --- a/internal/toolcall/toolcalls_test.go +++ b/internal/toolcall/toolcalls_test.go @@ -238,6 +238,23 @@ func TestParseToolCallsToleratesDSMLTrailingPipeTagTerminator(t *testing.T) { } } +func TestParseToolCallsToleratesDSMLTrailingNovelSeparatorTagTerminator(t *testing.T) { + text := strings.Join([]string{ + ``, + ` `, + ` `, + ` `, + ``, + }, "\n") + calls := ParseToolCalls(text, []string{"Bash"}) + if len(calls) != 1 { + t.Fatalf("expected one trailing-separator DSML call, got %#v", calls) + } + if calls[0].Name != "Bash" || calls[0].Input["command"] != "pwd" { + t.Fatalf("unexpected trailing-separator DSML parse result: %#v", calls[0]) + } +} + func TestParseToolCallsToleratesExtraLeadingLessThanBeforeDSML(t *testing.T) { text := `<<|DSML|tool_calls><<|DSML|invoke name="Bash"><<|DSML|parameter name="command">` calls := ParseToolCalls(text, []string{"Bash"}) diff --git a/tests/node/stream-tool-sieve.test.js b/tests/node/stream-tool-sieve.test.js index 22ec81f..0e55fca 100644 --- a/tests/node/stream-tool-sieve.test.js +++ b/tests/node/stream-tool-sieve.test.js @@ -247,6 +247,20 @@ test('parseToolCalls tolerates DSML trailing pipe tag terminator', () => { assert.deepEqual(calls[0].input, { command: 'find "/home" -type d', timeout: 10 }); }); +test('parseToolCalls tolerates DSML trailing novel separator tag terminator', () => { + const payload = [ + '', + ' ', + ' ', + ' ', + '', + ].join('\n'); + const calls = parseToolCalls(payload, ['Bash']); + assert.equal(calls.length, 1); + assert.equal(calls[0].name, 'Bash'); + assert.deepEqual(calls[0].input, { command: 'pwd' }); +}); + test('parseToolCalls tolerates extra leading less-than before DSML tags', () => { const payload = [ '<<|DSML|tool_calls>',