From 7a221b672e49dfae459b1af27210ba3f2b5419b6 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jul 2024 10:21:30 +0300 Subject: [PATCH] llama : use F32 precision in Qwen2 attention and no FA (#8412) --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index b19d786e2..ed77ed918 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -8134,7 +8134,7 @@ static struct ggml_tensor * llm_build_kqv( struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); cb(kq, "kq", il); - if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX) { + if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2) { // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847 ggml_mul_mat_set_prec(kq, GGML_PREC_F32);