root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-12 19:50:17 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

metal : try to improve batched decoding

Browse Source
This commit is contained in:
Georgi Gerganov 2023-12-01 21:47:42 +02:00
parent 3e68df8616
commit 3cb1c348b3
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ggml-metal.m
View File

@ -1083,7 +1083,7 @@ void ggml_metal_graph_compute(
// find the break-even point where the matrix-matrix kernel becomes more efficient compared
// to the matrix-vector kernel
int ne11_mm_min = 1;
int ne11_mm_min = src0t == GGML_TYPE_F16 ? 1 : 16;
#if 0
// the numbers below are measured on M2 Ultra for 7B and 13B models