fix(openai): keep a separate logprob for each multibyte token

lightllm/server/api_openai.py

@@ -885,7 +885,12 @@ async def stream_results() -> AsyncGenerator[bytes, None]:
                     model=request.model,
                     choices=[final_choice],
                 )
-                yield f"data: {_serialize_sse_chunk(final_chunk, _final_choice_nulls)}\n\n"
+                final_logprobs = None
+                if logprob_buffer is not None:
+                    content_tokens, reasoning_tokens = logprob_buffer.message_logprobs()
+                    if content_tokens or reasoning_tokens:
+                        final_logprobs = {"content": content_tokens, "reasoning_content": reasoning_tokens}
+                yield f"data: {_serialize_sse_chunk(final_chunk, _final_choice_nulls, logprobs=final_logprobs)}\n\n"
 
         reasoning_parser = get_env_start_args().reasoning_parser
         completion_tokens_details = None

lightllm/server/chat_logprob_buffer.py

@@ -12,7 +12,8 @@ class ChatLogprobBuffer:
     """按 token 序列识别分隔符，把可见文本的 logprob 放进正文或思维链。
 
     分隔符可以是一个或多个 token。凑齐才切换通道，分隔符本身不输出。
-    空增量的分数先累加，加到后面第一个非空文本上。生成结束时还没拼上的空增量丢掉。
+    每个采样位置单独一条 logprob。未完成的 UTF-8 字节文本为空，字符文本落在补齐的那个
+    token 上，分数不相加。特殊 token 的空文本不输出。
     """
 
     def __init__(
@@ -28,7 +29,6 @@ def __init__(
         self._separate_reasoning = separate_reasoning
         self._channel = "content"  # 当前输出通道：content、reasoning，或 commentary 的 drop。
         self._in_tool = False  # 是否处在工具调用开始和结束分隔符之间。
-        self._pending_logprob = 0.0  # 尚未拼成字符的空增量分数，等下一个非空文本一起输出。
         self._reasoning: list[dict[str, Any]] = []  # 分开思考时的思维链 logprob。
         self._content: list[dict[str, Any]] = []  # 正文 logprob；不分开思考时思维链也放这里。
         self._sequences: list[tuple[tuple[int, ...], str]] = []  # 分隔符 token id 序列和切换动作。
@@ -47,7 +47,6 @@ def record(self, text: str, metadata: dict[str, Any], finish_status: FinishStatu
             self._resolve_held(closing=False)
         if finish_status.is_finished():
             self._resolve_held(closing=True)
-            self._pending_logprob = 0.0
 
     def reasoning_logprobs(self) -> dict[str, Any] | None:
         tokens = self._drain(items=self._reasoning)
@@ -193,16 +192,11 @@ def _release(self, item: _Held) -> None:
                 self._save(text=text, score=score, metadata=metadata, token_id=token_id)
 
     def _save(self, text: str, score: float, metadata: dict[str, Any], token_id: int) -> None:
-        if not text:
-            # 未完成的 UTF-8 字节并进后面的字符。特殊 token 的空文本直接丢掉。
-            if not self._is_special(metadata=metadata, token_id=token_id):
-                self._pending_logprob += score
+        if not text and self._is_special(metadata=metadata, token_id=token_id):
             return
-        logprob = self._pending_logprob + score
-        self._pending_logprob = 0.0
         token = {
             "token": text,
-            "logprob": logprob,
+            "logprob": score,
             "bytes": None if "\ufffd" in text else list(text.encode("utf-8")),
             "top_logprobs": [
                 _candidate_logprob(tokenizer=self.tokenizer, token_id=candidate, logprob=value)

unit_tests/server/test_chat_output_logprobs.py

@@ -107,8 +107,11 @@ async def generate(prompt, params, *args, **kwargs):
     assert [entry["token"] for entry in reasoning] == (["think"] if separate_reasoning else [])
     expected_content = "你 yes" if separate_reasoning else "think你 yes"
     assert bytes(byte for entry in content for byte in entry["bytes"]).decode() == expected_content
-    assert [entry["token"] for entry in content] == (["你", " yes"] if separate_reasoning else ["think", "你", " yes"])
-    assert [entry["logprob"] for entry in content] == ([-0.75, -0.25] if separate_reasoning else [-0.25, -0.75, -0.25])
+    assert [entry["token"] for entry in content] == (
+        ["", "", "你", " yes"] if separate_reasoning else ["think", "", "", "你", " yes"]
+    )
+    expected_logprobs = [-0.25, -0.25, -0.25, -0.25] if separate_reasoning else [-0.25] * 5
+    assert [entry["logprob"] for entry in content] == expected_logprobs
     assert [entry["logprob"] for entry in reasoning] == ([-0.25] if separate_reasoning else [])
     expected = [{"token": " yes", "logprob": -0.125, "bytes": [32, 121, 101, 115]}] if topk > 0 else []
     assert all(entry["top_logprobs"] == expected for entry in content + reasoning)
@@ -287,13 +290,13 @@ async def test_chat_keeps_the_sampled_logprob(chat_response, stream, topk, rank)
 
 @pytest.mark.anyio
 @pytest.mark.parametrize("stream", [False, True])
-async def test_empty_increments_fuse_into_the_following_text(chat_response, stream):
+async def test_empty_increments_keep_their_own_logprob(chat_response, stream):
     tokenizer = make_tokenizer(["<0xE4>", "<0xBD>", "<0xA0>"])
     text, _, content, _ = await chat_response(tokenizer, [(0, ""), (1, ""), (2, "你")], stream=stream)
     assert text == "你"
-    assert [entry["token"] for entry in content] == ["你"]
-    assert content[0]["logprob"] == pytest.approx(-2.401099 * 3)
-    assert content[0]["bytes"] == list("你".encode())
+    assert [entry["token"] for entry in content] == ["", "", "你"]
+    assert [entry["logprob"] for entry in content] == pytest.approx([-2.401099, -2.401099, -2.401099])
+    assert [entry["bytes"] for entry in content] == [[], [], list("你".encode())]
 
 
 @pytest.mark.anyio
@@ -424,20 +427,22 @@ async def test_tool_delimiter_fused_into_another_token_is_rejected(chat_response
         await chat_response(tokenizer, list(enumerate(pieces)), stream=False, tools=tools)
 
 
-def test_empty_increments_fuse_and_trailing_ones_are_dropped():
+def test_empty_increments_keep_each_position_including_a_trailing_one():
     from lightllm.server.chat_logprob_buffer import ChatLogprobBuffer
 
     tokenizer = SimpleNamespace(decode=lambda ids, **kwargs: "")
     buffer = ChatLogprobBuffer(tokenizer, None, False, True)
     for token_id, text in enumerate(["", "", "你"]):
         buffer.record(text, {"id": token_id, "logprob": -0.25}, FinishStatus())
     content, _ = buffer.message_logprobs()
-    assert [token["token"] for token in content] == ["你"]
-    assert content[0]["logprob"] == -0.75
-    assert content[0]["bytes"] == list("你".encode())
+    assert [token["token"] for token in content] == ["", "", "你"]
+    assert [token["logprob"] for token in content] == [-0.25, -0.25, -0.25]
+    assert content[2]["bytes"] == list("你".encode())
 
     buffer.record("", {"id": 3, "logprob": -0.5}, FinishStatus(FinishStatus.FINISHED_STOP))
-    assert buffer.message_logprobs() == (None, None)
+    content, reasoning = buffer.message_logprobs()
+    assert [(token["token"], token["logprob"]) for token in content] == [("", -0.5)]
+    assert reasoning is None
 
 
 def test_reasoning_delimiter_must_be_one_token():