root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-09-23 05:26:19 +00:00
Code Issues Actions 16 Packages Projects Releases Wiki Activity

server : fix incorrect num_tokens_predicted (#3480)

Browse Source
This commit is contained in:
Jhen-Jie Hong 2023-10-05 09:02:55 -05:00 committed by GitHub
parent 8f3a642ec1
commit e8b8d32e86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
examples/server/server.cpp
View File

@ -504,9 +504,11 @@ struct llama_server_context
});
}
bool tg = true;
while (n_past < embd.size())
{
int n_eval = (int)embd.size() - n_past;
tg = n_eval == 1;
if (n_eval > params.n_batch)
{
n_eval = params.n_batch;
@ -633,7 +635,9 @@ struct llama_server_context
last_n_tokens.erase(last_n_tokens.begin());
last_n_tokens.push_back(result.tok);
num_tokens_predicted++;
if (tg) {
num_tokens_predicted++;
}
}
// add it to the context
@ -1124,8 +1128,6 @@ static json format_timings(llama_server_context &llama)
{
const auto timings = llama_get_timings(llama.ctx);
assert(timings.n_eval == ptrdiff_t(llama.num_tokens_predicted));
return json{
{"prompt_n", timings.n_p_eval},
{"prompt_ms", timings.t_p_eval_ms},