server: Propagate standby_timeout after it has been initialized

This commit is contained in:
johannes 2024-12-11 08:41:51 +01:00
parent 4fd58a8013
commit 7006dd784c

View File

@ -1430,10 +1430,6 @@ struct server_context {
// Necessary similarity of prompt for slot selection // Necessary similarity of prompt for slot selection
float slot_prompt_similarity = 0.0f; float slot_prompt_similarity = 0.0f;
server_context() {
queue_tasks.standby_timeout = params_base.standby_timeout;
}
~server_context() { ~server_context() {
if (ctx) { if (ctx) {
llama_free(ctx); llama_free(ctx);
@ -1485,6 +1481,8 @@ struct server_context {
n_ctx = llama_n_ctx(ctx); n_ctx = llama_n_ctx(ctx);
queue_tasks.standby_timeout = params.standby_timeout;
add_bos_token = llama_add_bos_token(model); add_bos_token = llama_add_bos_token(model);
has_eos_token = !llama_add_eos_token(model); has_eos_token = !llama_add_eos_token(model);