root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-11-15 07:19:53 +00:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

use reference quantization fns in AMX until moved to CPU backend

Browse Source 
ggml-ci
This commit is contained in:
slaren 2024-11-12 22:06:00 +01:00
parent 5cfaecd34c
commit dddf3771c2
3 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ggml/src/ggml-amx/ggml-amx.cpp
View File

@ -433,4 +433,8 @@ void ggml_backend_amx_set_n_threads(ggml_backend_t backend_amx, int n_threads) {
GGML_UNUSED(n_threads);
}
ggml_backend_reg_t ggml_backend_amx_reg(void) {
return nullptr;
}
#endif

7
ggml/src/ggml-amx/mmq.cpp
View File

@ -496,19 +496,20 @@ inline void from_float(const float * x, char * vy, int64_t k);
template <>
inline void from_float<block_q8_0>(const float * x, char * vy, int64_t k) {
quantize_row_q8_0(x, vy, k);
// FIXME: using unoptimized reference impl until moved to CPU backend
quantize_row_q8_0_ref(x, (block_q8_0 *)vy, k);
}
template <>
inline void from_float<block_q8_1>(const float * x, char * vy, int64_t k) {
quantize_row_q8_1(x, vy, k);
quantize_row_q8_1_ref(x, (block_q8_1 *)vy, k);
}
template <>
inline void from_float<block_q8_K>(const float * x, char * vy, int64_t k) {
#if 1
// TODO: this is reference impl!
quantize_row_q8_K(x, vy, k);
quantize_row_q8_K_ref(x, (block_q8_K *)vy, k);
#else
quantize_row_q8_K_vnni(x, vy, k);
#endif

8
ggml/src/ggml-backend-reg.cpp
View File

@ -31,10 +31,6 @@
#include "ggml-rpc.h"
#endif
#ifndef __AMX_INT8__
#undef GGML_USE_AMX
#endif
#ifdef GGML_USE_AMX
# include "ggml-amx.h"
#endif
@ -84,6 +80,10 @@ struct ggml_backend_registry {
}
void register_backend(ggml_backend_reg_t reg) {
if (!reg) {
return;
}
#ifndef NDEBUG
GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
__func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));