root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 02:44:36 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

llama : remember and restore kv cache data pointers (#1104)

Browse Source 
because their value is stored in buf and overwritten by memcpy
This commit is contained in:
xaedes 2023-04-21 17:25:21 +02:00 committed by GitHub
parent 1bfc153e2f
commit 8687c1f258
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llama.cpp
View File

@ -2092,7 +2092,11 @@ void llama_set_kv_cache(
int n_token_count) { int n_token_count) {
// Make sure we have the same kv cache setup // Make sure we have the same kv cache setup
LLAMA_ASSERT(ctx->model.kv_self.buf.size == n_size); LLAMA_ASSERT(ctx->model.kv_self.buf.size == n_size);
void * k_data = ctx->model.kv_self.k->data; // remember data pointers
void * v_data = ctx->model.kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
memcpy(ctx->model.kv_self.buf.addr, kv_cache, n_size); memcpy(ctx->model.kv_self.buf.addr, kv_cache, n_size);
ctx->model.kv_self.k->data = k_data; // restore correct data pointers
ctx->model.kv_self.v->data = v_data;
ctx->model.kv_self.n = n_token_count; ctx->model.kv_self.n = n_token_count;
} }