feat: enhance content filtering, token usage tracking, and stream error handling in chat-stream modules

tests/compat/expected/sse_content_filter_status.json

@@ -0,0 +1,8 @@
+{
+  "parts": [],
+  "finished": true,
+  "new_type": "text",
+  "content_filter": true,
+  "output_tokens": 77,
+  "error_message": ""
+}

tests/compat/expected/sse_leaked_content_filter.json

@@ -0,0 +1,7 @@
+{
+  "parts": [
+    {"text": "正常输出", "type": "text"}
+  ],
+  "finished": false,
+  "new_type": "text"
+}

tests/compat/expected/token_cases.json

@@ -1,6 +1,8 @@
 {
   "cases": [
     {"name": "ascii_short", "tokens": 1},
+    {"name": "whitespace_only", "tokens": 1},
+    {"name": "newline_only", "tokens": 1},
     {"name": "cjk", "tokens": 3},
     {"name": "mixed", "tokens": 4}
   ]

tests/compat/fixtures/sse_chunks/content_filter_status.json

@@ -0,0 +1,11 @@
+{
+  "chunk": {
+    "p": "response",
+    "v": [
+      {"p": "status", "v": "CONTENT_FILTER"},
+      {"p": "accumulated_token_usage", "v": 77}
+    ]
+  },
+  "thinking_enabled": false,
+  "current_type": "text"
+}

tests/compat/fixtures/sse_chunks/leaked_content_filter.json

@@ -0,0 +1,8 @@
+{
+  "chunk": {
+    "p": "response/content",
+    "v": "正常输出CONTENT_FILTER你好，这个问题我暂时无法回答"
+  },
+  "thinking_enabled": false,
+  "current_type": "text"
+}

tests/compat/fixtures/token_cases.json

@@ -1,6 +1,8 @@
 {
   "cases": [
     {"name": "ascii_short", "text": "abcd"},
+    {"name": "whitespace_only", "text": "   "},
+    {"name": "newline_only", "text": "\n"},
     {"name": "cjk", "text": "你好世界"},
     {"name": "mixed", "text": "Hello 你好世界"}
   ]

tests/node/chat-stream.test.js

@@ -17,6 +17,8 @@ const {
   normalizePreparedToolNames,
   boolDefaultTrue,
   filterIncrementalToolCallDeltasByAllowed,
+  buildUsage,
+  estimateTokens,
   shouldSkipPath,
   isNodeStreamSupportedPath,
   extractPathname,
@@ -245,6 +247,84 @@ test('parseChunkForContent strips reference markers from fragment content', () =
   assert.deepEqual(parsed.parts, [{ text: '广州天气  多云', type: 'text' }]);
 });
 
+test('parseChunkForContent detects content_filter status and carries output tokens', () => {
+  const chunk = {
+    p: 'response',
+    v: [
+      { p: 'status', v: 'CONTENT_FILTER' },
+      { p: 'accumulated_token_usage', v: 77 },
+    ],
+  };
+  const parsed = parseChunkForContent(chunk, false, 'text');
+  assert.equal(parsed.parsed, true);
+  assert.equal(parsed.finished, true);
+  assert.equal(parsed.contentFilter, true);
+  assert.equal(parsed.outputTokens, 77);
+  assert.deepEqual(parsed.parts, []);
+});
+
+test('parseChunkForContent keeps error branches distinct from content_filter status', () => {
+  const chunk = {
+    error: { message: 'boom' },
+    code: 'content_filter',
+    accumulated_token_usage: 88,
+  };
+  const parsed = parseChunkForContent(chunk, false, 'text');
+  assert.equal(parsed.parsed, true);
+  assert.equal(parsed.finished, true);
+  assert.equal(parsed.contentFilter, false);
+  assert.equal(parsed.errorMessage.length > 0, true);
+  assert.equal(parsed.outputTokens, 0);
+  assert.deepEqual(parsed.parts, []);
+});
+
+test('parseChunkForContent preserves output tokens on FINISHED lines', () => {
+  const parsed = parseChunkForContent(
+    { p: 'response/status', v: 'FINISHED', accumulated_token_usage: 190 },
+    false,
+    'text',
+  );
+  assert.equal(parsed.parsed, true);
+  assert.equal(parsed.finished, true);
+  assert.equal(parsed.contentFilter, false);
+  assert.equal(parsed.outputTokens, 190);
+  assert.deepEqual(parsed.parts, []);
+});
+
+test('parseChunkForContent strips leaked CONTENT_FILTER suffix and preserves line breaks', () => {
+  const leaked = parseChunkForContent(
+    { p: 'response/content', v: '正常输出CONTENT_FILTER你好，这个问题我暂时无法回答' },
+    false,
+    'text',
+  );
+  assert.deepEqual(leaked.parts, [{ text: '正常输出', type: 'text' }]);
+
+  const newlineTail = parseChunkForContent(
+    { p: 'response/content', v: 'line1\nCONTENT_FILTERblocked' },
+    false,
+    'text',
+  );
+  assert.deepEqual(newlineTail.parts, [{ text: 'line1\n', type: 'text' }]);
+
+  const newlineOnly = parseChunkForContent(
+    { p: 'response/content', v: '\nCONTENT_FILTERblocked' },
+    false,
+    'text',
+  );
+  assert.deepEqual(newlineOnly.parts, [{ text: '\n', type: 'text' }]);
+});
+
+test('estimateTokens preserves whitespace-only strings and buildUsage accepts output token overrides', () => {
+  assert.equal(estimateTokens('   '), 1);
+  assert.equal(estimateTokens('\n'), 1);
+
+  const usage = buildUsage('abcd', 'ef', 'gh', 99);
+  assert.equal(usage.prompt_tokens, 1);
+  assert.equal(usage.completion_tokens, 99);
+  assert.equal(usage.total_tokens, 100);
+  assert.equal(usage.completion_tokens_details.reasoning_tokens, 1);
+});
+
 test('shouldSkipPath skips dynamic response/fragments/*/status paths only', () => {
   assert.equal(shouldSkipPath('response/fragments/-16/status'), true);
   assert.equal(shouldSkipPath('response/fragments/8/status'), true);

tests/node/js_compat_test.js

@@ -30,6 +30,9 @@ test('js compat: sse fixtures', () => {
     assert.deepEqual(got.parts, expected.parts, `${name}: parts mismatch`);
     assert.equal(got.finished, expected.finished, `${name}: finished mismatch`);
     assert.equal(got.newType, expected.new_type, `${name}: newType mismatch`);
+    assert.equal(Boolean(got.contentFilter), Boolean(expected.content_filter), `${name}: contentFilter mismatch`);
+    assert.equal(Number(got.outputTokens || 0), Number(expected.output_tokens || 0), `${name}: outputTokens mismatch`);
+    assert.equal(got.errorMessage || '', expected.error_message || '', `${name}: errorMessage mismatch`);
   }
 });