root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 19:04:35 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

ggml : update softmax n_task calculation (#5126)

Browse Source 
updated the n_task calculation to use max number of
threads possible. This has improved the prompt eval
performance by around 5% for DOT kernels and by
around 10% for MMLA kernels on AWS Graviton3.
This commit is contained in:
snadampal 2024-01-26 11:17:59 -06:00 committed by GitHub
parent 5f1925a8ce
commit 7032f4f634
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ggml.c
View File

@ -16597,7 +16597,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
} break; } break;
case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX:
{ {
n_tasks = MIN(MIN(4, n_threads), ggml_nrows(node->src[0])); n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
} break; } break;
case GGML_OP_CONV_TRANSPOSE_1D: case GGML_OP_CONV_TRANSPOSE_1D:
{ {