root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-13 04:00:16 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Add default value for attention and final logit softcap value

Browse Source
This commit is contained in:
Andrei Betlen 2024-06-29 01:10:55 -04:00
parent 8edf73a729
commit bb7159927d
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llama.cpp
View File

@ -2103,8 +2103,8 @@ struct llama_hparams {
float f_norm_eps; float f_norm_eps;
float f_norm_rms_eps; float f_norm_rms_eps;
float f_attn_logit_softcapping; float f_attn_logit_softcapping = 50.0f;
float f_final_logit_softcapping; float f_final_logit_softcapping = 30.0f;
float rope_attn_factor = 1.0f; float rope_attn_factor = 1.0f;
float rope_freq_base_train; float rope_freq_base_train;
@ -4710,8 +4710,8 @@ static void llm_load_hparams(
case LLM_ARCH_GEMMA2: case LLM_ARCH_GEMMA2:
{ {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping); ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping, false);
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping); ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
hparams.attn_soft_cap = true; hparams.attn_soft_cap = true;
switch (hparams.n_layer) { switch (hparams.n_layer) {