root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-26 03:14:35 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

llama : add missing kv clear in llama_beam_search (#6664)

Browse Source
This commit is contained in:
David Renshaw 2024-04-14 15:24:15 -04:00 committed by GitHub
parent 04fbc5f23e
commit 1958f7e06c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
llama.cpp
View File

@ -13063,6 +13063,11 @@ struct llama_beam_search_data {
} }
llama_logit_info logit_info(ctx); llama_logit_info logit_info(ctx);
std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams); std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams);
// Clear the kv slot so that other beams may try different tokens at this position. The llama_decode()
// call in loop() will conclusively fill in the kv slot once the beams converge at this position.
llama_kv_cache_seq_rm(ctx, 0, n_past, -1);
size_t i=0; size_t i=0;
if (next_beams.size() < n_beams) { if (next_beams.size() < n_beams) {
for (; next_beams.size() < n_beams ; ++i) { for (; next_beams.size() < n_beams ; ++i) {