root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-27 03:44:35 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

server : bach has to be allocated for n_parallel sequences

Browse Source
This commit is contained in:
Georgi Gerganov 2023-10-20 20:42:45 +03:00
parent 6b2437e32d
commit 113dd60005
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/server/server.cpp
View File

@ -631,7 +631,9 @@ struct llama_server_context
LOG_TEE(" -> Slot %i - max context: %i\n", slot.id, max_ctx_per_slot);
slots.push_back(slot);
}
batch = llama_batch_init(n_ctx, 0, 1);
batch = llama_batch_init(n_ctx, 0, params.n_parallel);
// empty system prompt
system_prompt = "";
num_tokens_system = 0;