root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-07 09:11:46 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

speculative : limit batch size to llama_n_batch

Browse Source
This commit is contained in:
Georgi Gerganov 2024-10-20 20:15:59 +03:00
parent 67d18498d3
commit 90ab8a10d5
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/speculative/speculative.cpp
View File

@ -195,8 +195,8 @@ int main(int argc, char ** argv) {
drafts[s].smpl = common_sampler_init(model_dft, params.sparams);
}
llama_batch batch_dft = llama_batch_init(llama_n_ctx(ctx_dft), 0, 1);
llama_batch batch_tgt = llama_batch_init(llama_n_ctx(ctx_tgt), 0, n_seq_dft);
llama_batch batch_dft = llama_batch_init(llama_n_batch(ctx_dft), 0, 1);
llama_batch batch_tgt = llama_batch_init(llama_n_batch(ctx_tgt), 0, n_seq_dft);
const auto t_dec_start = ggml_time_us();