mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-11-11 21:39:52 +00:00
fbf1ddec69
Signed-off-by: Jared Van Bortel <jared@nomic.ai> Co-authored-by: niansa <anton-sa@web.de> Co-authored-by: Adam Treat <treat.adam@gmail.com> Co-authored-by: Aaron Miller <apage43@ninjawhale.com> Co-authored-by: ToKiNoBug <tokinobug@163.com> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> Co-authored-by: slaren <slarengh@gmail.com>
103 lines
3.5 KiB
Plaintext
103 lines
3.5 KiB
Plaintext
#extension GL_EXT_shader_16bit_storage: require
|
|
#extension GL_EXT_shader_8bit_storage: require
|
|
#extension GL_EXT_shader_explicit_arithmetic_types_float16: require
|
|
#extension GL_EXT_shader_explicit_arithmetic_types_int8: require
|
|
#extension GL_EXT_shader_explicit_arithmetic_types_int16: require
|
|
#extension GL_EXT_control_flow_attributes: enable
|
|
#extension GL_KHR_shader_subgroup_arithmetic : require
|
|
#extension GL_EXT_debug_printf : enable
|
|
|
|
#define QK4_0 32
|
|
#define QK4_1 32
|
|
|
|
#define GELU_COEF_A 0.044715
|
|
#define SQRT_2_OVER_PI 0.79788456080286535587989211986876
|
|
#define TWOPI_F 6.283185307179586f
|
|
|
|
#define QK_K 256
|
|
|
|
#define u8BufToU16(buf, idx) (((uint16_t(buf[idx + 1]) << 8)) | buf[idx])
|
|
#define u8BufToFloat16(buf, idx) uint16BitsToHalf u8BufToU16(buf, idx)
|
|
#define u8BufToU32(buf, idx) (((uint32_t u8BufToU16(buf, idx + 2) << 8 | buf[idx + 1]) << 8) | buf[idx])
|
|
#define u8BufToFloat(buf, idx) uintBitsToFloat u8BufToU32(buf, idx)
|
|
|
|
#define sizeof_block_q4_0 0x12
|
|
struct block_q4_0 {
|
|
float16_t d;
|
|
uint8_t qs[QK4_0 / 2];
|
|
};
|
|
mat4 dequantize_q4_0(const block_q4_0 xb, uint il) {
|
|
const float d1 = il != 0 ? (xb.d / 16.f) : xb.d;
|
|
const float d2 = d1 / 256.f;
|
|
const float md = -8.f * xb.d;
|
|
const uint16_t mask0 = il != 0 ? uint16_t(0x00F0) : uint16_t(0x000F);
|
|
const uint16_t mask1 = mask0 << 8;
|
|
|
|
mat4 reg;
|
|
for (int i=0;i<8;i++) {
|
|
uint16_t b = (uint16_t(xb.qs[2 * i + 1]) << 8) | uint16_t(xb.qs[2 * i]);
|
|
reg[i/2][2*(i%2)+0] = d1 * (b & mask0) + md;
|
|
reg[i/2][2*(i%2)+1] = d2 * (b & mask1) + md;
|
|
}
|
|
return reg;
|
|
}
|
|
|
|
#define sizeof_block_q4_1 0x14
|
|
struct block_q4_1 {
|
|
float16_t d;
|
|
float16_t m;
|
|
uint8_t qs[QK4_1 / 2];
|
|
};
|
|
mat4 dequantize_q4_1(const block_q4_1 xb, uint il) {
|
|
const float d1 = il != 0 ? (xb.d / 16.f) : xb.d;
|
|
const float d2 = d1 / 256.f;
|
|
const float m = xb.m;
|
|
const uint16_t mask0 = il != 0 ? uint16_t(0x00F0) : uint16_t(0x000F);
|
|
const uint16_t mask1 = mask0 << 8;
|
|
|
|
mat4 reg;
|
|
for (int i=0;i<8;i++) {
|
|
uint16_t b = (uint16_t(xb.qs[2 * i + 1]) << 8) | uint16_t(xb.qs[2 * i]);
|
|
reg[i/2][2*(i%2)+0] = ((b & mask0) * d1) + m;
|
|
reg[i/2][2*(i%2)+1] = ((b & mask1) * d2) + m;
|
|
}
|
|
return reg;
|
|
}
|
|
|
|
#define sizeof_block_q6_k 210
|
|
struct block_q6_k {
|
|
uint8_t ql[QK_K/2]; // quants, lower 4 bits
|
|
uint8_t qh[QK_K/4]; // quants, upper 2 bits
|
|
int8_t scales[QK_K/16]; // scales, quantized with 8 bits
|
|
float16_t d; // super-block scale
|
|
};
|
|
mat4 dequantize_q6_k(const block_q6_k xb, uint il) {
|
|
const float16_t d_all = xb.d;
|
|
|
|
const uint qlIndex = 64*(il/8) + 32*((il/2)&1) + 16*(il&1);
|
|
const uint qhIndex = 32*(il/8) + 16*(il&1);
|
|
float16_t sc = xb.scales[(il%2) + 2 * ((il/2))];
|
|
il = (il/2) & 3;
|
|
|
|
const uint16_t kmask1 = il>1 ? uint16_t(il>2 ? 192 : 48) : uint16_t(il>0 ? 12 : 3);
|
|
const uint16_t kmask2 = il>1 ? uint8_t(0xF0) : uint8_t(0x0F);
|
|
const float16_t coef = il>1 ? float16_t(1.f/16.f) : float16_t(1.f);
|
|
const float16_t ml = float16_t(d_all * sc * 32.f);
|
|
const float16_t dl = float16_t(d_all * sc * coef);
|
|
mat4 reg;
|
|
for (int i = 0; i < 16; ++i) {
|
|
const float16_t q = (il&1) != 0 ? ((xb.ql[qlIndex + i] & kmask2) | ((xb.qh[qhIndex + i] & kmask1) << 2))
|
|
: ((xb.ql[qlIndex + i] & kmask2) | ((xb.qh[qhIndex + i] & kmask1) << 4));
|
|
reg[i/4][i%4] = dl * q - ml;
|
|
}
|
|
return reg;
|
|
}
|
|
|
|
|
|
#define QK8_0 32
|
|
// struct block_q8_0 {
|
|
// float16_t d; // delta
|
|
// int8_t qs[QK8_0]; // quants
|
|
// };
|
|
#define sizeof_block_q8_0 34
|