root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-27 03:44:35 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

llama : minor

Browse Source
This commit is contained in:
Georgi Gerganov 2023-10-30 11:34:47 +02:00
parent f39e6075cf
commit 792d1a1b16
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 9 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
llama.cpp
View File

@ -3503,8 +3503,7 @@ static struct ggml_cgraph * llm_build_llama(
// norm // norm
cur = llm_build_norm(ctx0, inpL, cur = llm_build_norm(ctx0, inpL,
model.layers[il].attn_norm, model.layers[il].attn_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, il); LLM_NORM_RMS, norm_rms_eps, cb, il);
cb(cur, "attn_norm", il); cb(cur, "attn_norm", il);
@ -3540,8 +3539,7 @@ static struct ggml_cgraph * llm_build_llama(
// feed-forward network // feed-forward network
{ {
cur = llm_build_norm(ctx0, inpFF, cur = llm_build_norm(ctx0, inpFF,
model.layers[il].ffn_norm, model.layers[il].ffn_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, il); LLM_NORM_RMS, norm_rms_eps, cb, il);
cb(cur, "ffn_norm", il); cb(cur, "ffn_norm", il);
@ -3563,8 +3561,7 @@ static struct ggml_cgraph * llm_build_llama(
cur = inpL; cur = inpL;
cur = llm_build_norm(ctx0, cur, cur = llm_build_norm(ctx0, cur,
model.output_norm, model.output_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, -1); LLM_NORM_RMS, norm_rms_eps, cb, -1);
cb(cur, "result_norm", -1); cb(cur, "result_norm", -1);
@ -3661,8 +3658,7 @@ static struct ggml_cgraph * llm_build_baichaun(
struct ggml_tensor * inpSA = inpL; struct ggml_tensor * inpSA = inpL;
cur = llm_build_norm(ctx0, inpL, cur = llm_build_norm(ctx0, inpL,
model.layers[il].attn_norm, model.layers[il].attn_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, il); LLM_NORM_RMS, norm_rms_eps, cb, il);
cb(cur, "attn_norm", il); cb(cur, "attn_norm", il);
@ -3709,8 +3705,7 @@ static struct ggml_cgraph * llm_build_baichaun(
// feed-forward network // feed-forward network
{ {
cur = llm_build_norm(ctx0, inpFF, cur = llm_build_norm(ctx0, inpFF,
model.layers[il].ffn_norm, model.layers[il].ffn_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, il); LLM_NORM_RMS, norm_rms_eps, cb, il);
cb(cur, "ffn_norm", il); cb(cur, "ffn_norm", il);
@ -3732,8 +3727,7 @@ static struct ggml_cgraph * llm_build_baichaun(
cur = inpL; cur = inpL;
cur = llm_build_norm(ctx0, cur, cur = llm_build_norm(ctx0, cur,
model.output_norm, model.output_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, -1); LLM_NORM_RMS, norm_rms_eps, cb, -1);
cb(cur, "result_norm", -1); cb(cur, "result_norm", -1);
@ -4394,8 +4388,7 @@ static struct ggml_cgraph * llm_build_refact(
struct ggml_tensor * inpSA = inpL; struct ggml_tensor * inpSA = inpL;
cur = llm_build_norm(ctx0, inpL, cur = llm_build_norm(ctx0, inpL,
model.layers[il].attn_norm, model.layers[il].attn_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, il); LLM_NORM_RMS, norm_rms_eps, cb, il);
cb(cur, "attn_norm", il); cb(cur, "attn_norm", il);
@ -4430,8 +4423,7 @@ static struct ggml_cgraph * llm_build_refact(
// feed-forward network // feed-forward network
{ {
cur = llm_build_norm(ctx0, inpFF, cur = llm_build_norm(ctx0, inpFF,
model.layers[il].ffn_norm, model.layers[il].ffn_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, il); LLM_NORM_RMS, norm_rms_eps, cb, il);
cb(cur, "ffn_norm", il); cb(cur, "ffn_norm", il);
@ -4453,8 +4445,7 @@ static struct ggml_cgraph * llm_build_refact(
cur = inpL; cur = inpL;
cur = llm_build_norm(ctx0, cur, cur = llm_build_norm(ctx0, cur,
model.output_norm, model.output_norm, NULL,
NULL,
LLM_NORM_RMS, norm_rms_eps, cb, -1); LLM_NORM_RMS, norm_rms_eps, cb, -1);
cb(cur, "result_norm", -1); cb(cur, "result_norm", -1);