root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 10:54:36 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

ggml : fix AVX build + update to new Q8_0 format

Browse Source
This commit is contained in:
Georgi Gerganov 2023-04-22 11:08:12 +03:00
parent 955ef9a5d5
commit 872c365a91
3 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Makefile
View File

@ -74,13 +74,17 @@ endif
# feel free to update the Makefile for your architecture and send a pull request or issue # feel free to update the Makefile for your architecture and send a pull request or issue
ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686)) ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686))
# Use all CPU extensions that are available: # Use all CPU extensions that are available:
CFLAGS += -march=native -mtune=native CFLAGS += -march=native -mtune=native
CXXFLAGS += -march=native -mtune=native CXXFLAGS += -march=native -mtune=native
# Usage AVX-only
#CFLAGS += -mfma -mf16c -mavx
#CXXFLAGS += -mfma -mf16c -mavx
endif endif
ifneq ($(filter ppc64%,$(UNAME_M)),) ifneq ($(filter ppc64%,$(UNAME_M)),)
POWER9_M := $(shell grep "POWER9" /proc/cpuinfo) POWER9_M := $(shell grep "POWER9" /proc/cpuinfo)
ifneq (,$(findstring POWER9,$(POWER9_M))) ifneq (,$(findstring POWER9,$(POWER9_M)))
CFLAGS += -mcpu=power9 CFLAGS += -mcpu=power9
CXXFLAGS += -mcpu=power9 CXXFLAGS += -mcpu=power9
endif endif
# Require c++23's std::byteswap for big-endian support. # Require c++23's std::byteswap for big-endian support.
@ -114,7 +118,7 @@ ifdef LLAMA_GPROF
CXXFLAGS += -pg CXXFLAGS += -pg
endif endif
ifneq ($(filter aarch64%,$(UNAME_M)),) ifneq ($(filter aarch64%,$(UNAME_M)),)
CFLAGS += -mcpu=native CFLAGS += -mcpu=native
CXXFLAGS += -mcpu=native CXXFLAGS += -mcpu=native
endif endif
ifneq ($(filter armv6%,$(UNAME_M)),) ifneq ($(filter armv6%,$(UNAME_M)),)

12
ggml.c
View File

@ -468,6 +468,14 @@ static inline int hsum_i32_8(const __m256i a) {
return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32)); return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
} }
// horizontally add 4 int32_t
static inline int hsum_i32_4(const __m128i a) {
const __m128i hi64 = _mm_unpackhi_epi64(a, a);
const __m128i sum64 = _mm_add_epi32(hi64, a);
const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
}
#if __AVX2__ || __AVX512F__ #if __AVX2__ || __AVX512F__
// Unpack 32 4-bit fields into 32 bytes // Unpack 32 4-bit fields into 32 bytes
// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval // The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
@ -1381,7 +1389,6 @@ static void quantize_row_q8_0(const float * restrict x, void * restrict vy, int
y[i].s1 = d * sum1; y[i].s1 = d * sum1;
} }
#elif defined(__AVX2__) || defined(__AVX__) #elif defined(__AVX2__) || defined(__AVX__)
// TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
for (int i = 0; i < nb; i++) { for (int i = 0; i < nb; i++) {
// Load elements into 4 AVX vectors // Load elements into 4 AVX vectors
__m256 v0 = _mm256_loadu_ps( x ); __m256 v0 = _mm256_loadu_ps( x );
@ -1460,7 +1467,8 @@ static void quantize_row_q8_0(const float * restrict x, void * restrict vy, int
// Compute the sum of the quants and set y[i].s // Compute the sum of the quants and set y[i].s
const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3)); const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));
const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7)); const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));
y[i].s = d * hsum_i32_8(_mm256_set_m128i(s1, s0)); y[i].s0 = d * hsum_i32_4(s0);
y[i].s1 = d * hsum_i32_4(s1);
// Convert int32 to int16 // Convert int32 to int16
ni0 = _mm_packs_epi32( ni0, ni1 ); ni0 = _mm_packs_epi32( ni0, ni1 );

6
llama.cpp
View File

@ -68,7 +68,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
{ MODEL_65B, 512ull * MB }, { MODEL_65B, 512ull * MB },
}; };
return _MEM_REQ_SCRATCH1; return _MEM_REQ_SCRATCH1;
}; }
// 2*n_embd*n_ctx*n_layer*sizeof(float16) // 2*n_embd*n_ctx*n_layer*sizeof(float16)
static const std::map<e_model, size_t> & MEM_REQ_KV_SELF() static const std::map<e_model, size_t> & MEM_REQ_KV_SELF()
@ -80,7 +80,7 @@ static const std::map<e_model, size_t> & MEM_REQ_KV_SELF()
{ MODEL_65B, 5120ull * MB }, { MODEL_65B, 5120ull * MB },
}; };
return _MEM_REQ_KV_SELF; return _MEM_REQ_KV_SELF;
}; }
// this is mostly needed for temporary mul_mat buffers to dequantize the data // this is mostly needed for temporary mul_mat buffers to dequantize the data
// not actually needed if BLAS is disabled // not actually needed if BLAS is disabled
@ -93,7 +93,7 @@ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
{ MODEL_65B, 1536ull * MB }, { MODEL_65B, 1536ull * MB },
}; };
return _MEM_REQ_EVAL; return _MEM_REQ_EVAL;
}; }
// default hparams (LLaMA 7B) // default hparams (LLaMA 7B)
struct llama_hparams { struct llama_hparams {