root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-24 10:24:35 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Work around for recalculating logits in cached prompts (Fixes #1585) (#1609)

Browse Source 
* Work around for recalculating logits in cached prompts
This commit is contained in:
DannyDaemonic 2023-05-29 05:13:40 -07:00 committed by GitHub
parent 0e730dd23b
commit 248367605e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/main/main.cpp
View File

@ -360,6 +360,12 @@ int main(int argc, char ** argv) {
} }
} }
if (i > 0) { if (i > 0) {
// check if we've used up all the prompt but not all cached tokens
if (embd.size() == i && n_session_consumed < (int) session_tokens.size()) {
// force revaluation of the last token to recalculate logits
i--;
n_past--;
}
embd.erase(embd.begin(), embd.begin() + i); embd.erase(embd.begin(), embd.begin() + i);
} }
} }