darthrubik/llama-cpp
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

server : fix usage.completion_tokens undercount with n > 1 (#29486)
Copilot Setup Steps / copilot-setup-steps (push) Has been cancelled
Check Pre-Tokenizer Hashes / pre-tokenizer-hashes (push) Has been cancelled
Python check requirements.txt / check-requirements (push) Has been cancelled
Python Type-Check / python type-check (push) Has been cancelled
Update Operations Documentation / update-ops-docs (push) Has been cancelled

Browse Source 
Assisted-by: Copilot
This commit is contained in:
Unbug Lee
2026-10-21 07:19:54 +08:00
committed by GitHub
parent 5c0b49a23b
commit 9a3b91c680
2 changed files with 14 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+10
View File
@@ -5277,10 +5277,20 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
res->ok(arr[0]);
} else if (res_type == TASK_RESPONSE_TYPE_OAI_CHAT || res_type == TASK_RESPONSE_TYPE_OAI_CMPL) {
// if multiple results in OAI format, we need to re-format them
json & usage = arr[0]["usage"];
json & choices = arr[0]["choices"];
for (size_t i = 1; i < arr.size(); i++) {
choices.push_back(std::move(arr[i]["choices"][0]));
}
// OAI counts generated tokens across all choices
int64_t n_completion_tokens = 0;
for (auto & item : arr) {
n_completion_tokens += item["usage"]["completion_tokens"].get<int64_t>();
}
usage["completion_tokens"] = n_completion_tokens;
usage["total_tokens"] = usage["prompt_tokens"].get<int64_t>() + n_completion_tokens;
res->ok(arr[0]);
} else {
// multi-results, non-OAI compat
tools/server/tests/unit/test_chat_completion.py
+4
View File
@@ -587,6 +587,10 @@ def test_chat_completions_multiple_choices():
for choice in res.body["choices"]:
assert "assistant" == choice["message"]["role"]
assert choice["finish_reason"] == "length"
# each choice generates max_tokens tokens, usage counts them all
# ref: https://github.com/ggml-org/llama.cpp/issues/29451
assert res.body["usage"]["completion_tokens"] == 16
assert res.body["usage"]["total_tokens"] == res.body["usage"]["prompt_tokens"] + 16
def test_chat_completions_token_count():