root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-28 12:24:35 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

cuda : avoid warp_reduce for smax

Browse Source
This commit is contained in:
Georgi Gerganov 2024-02-03 13:17:47 +02:00
parent b68a112204
commit b150abe83e
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ggml-cuda.cu
View File

@ -6621,7 +6621,6 @@ static __global__ void flash_attn_ext_f16(
M[j] = __hmax(M[j], s); M[j] = __hmax(M[j], s);
} }
smax = warp_reduce_max(smax);
M[j] = warp_reduce_max(M[j]); M[j] = warp_reduce_max(M[j]);
const half ms = __hisinf(m) == -1 ? __float2half(0.0f) : hexp(m - M[j]); const half ms = __hisinf(m) == -1 ? __float2half(0.0f) : hexp(m - M[j]);
@ -6649,6 +6648,8 @@ static __global__ void flash_attn_ext_f16(
} }
} }
smax = warp_reduce_max(smax);
// skip -INF blocks // skip -INF blocks
if (__hisinf(smax) == -1) { if (__hisinf(smax) == -1) {
continue; continue;