mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-11-11 21:39:52 +00:00
55c1b2a3bb
* iq1_m: basics * iq1_m: basics-2 * iq1_m: CUDA dequantize works Very 1st shot I get PPL = 9.76 for LLaMA-v2-7B. * iq1_m: separate shifts for each group of 8 in a block We get PPL(LLaMA-v2-7B ) = 9.2810 PPL(LLaMA-v2-13B) = 6.8105 Not bad, but slightly higher than sqrt(PPL(IQ1_S) * PPL(IQ2_XXS)) which is the expected outcome given that IQ1_M is halfway between IQ1_S and IQ2_XXS in terms of bpw. From this, we would expect PPL = 9.14 for LLaMA-v2-7B PPL = 6.63 for LLaMA-v2-13B * iq1_m: go to 3-bit scales There is slight increase in PPL, but the 0.0625 bpw reduction in size is totally worth it. We now have PPL(LLaMA-v2-7B ) = 9.4469 at 1.96 bpw PPL(LLaMA-v2-13B) = 6.8717 at 1.93 bpw PPL(LLaMA-v2-70B) = 4.8568 at 1.85 bpw * iq1_m: scalar dot product * iq1_m: AVX2 dot product * iq1_m: very slightly faster AVX2 dot product * iq1_m: ARM_NEON dot product Works, but very slow (10.5 t/s) * iq1_m: Metal - dequantize works, dot product does not * iq1_m: Metal now works About the same performance as iq1_s. * iq1_m: minor * iq1_m: checking pure iq1_m quantization It is pretty bad: PPL(LLaMA-v2-7B) = 34 if we quantize output.weight with Q4_K. * iiq1_m: slightly faster ARM_NEON dot product 10.5 t/s -> 11.65 t/s * iq1_m: faster ARM_NEON dot product 11.65 t/s -> 14.9 t/s * iq1_m: another minor ARM_NEON dot product improvement 14.9 -> 15.0 t/s * iq1_m: small PPL improvement via super-block scale adjustment After quantizing block scales redo the super-block scale fit. PPL(LLaMA-v2-7B ) = 9.3346 PPL(LLaMA-v2-13B) = 6.8419 PPL(LLaMA-v2-70B) = 4.8294 PPL(Mistral-7B ) = 8.1624 * iq1_m: adapt to CUDA refactoring * iq1_m: remove unused variable We have progressed to warnings being errors. * iq1_m: add to backend-ops tests * iq1_m: fix Windows ARM * iq1_m: use common definition of iq1m_scale_t * cuda: assert -> NO_DEVICE_CODE * iq1_M: PR comments --------- Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
845 lines
26 KiB
Python
845 lines
26 KiB
Python
from __future__ import annotations
|
|
|
|
import sys
|
|
from enum import Enum, IntEnum, auto
|
|
from typing import Any
|
|
|
|
#
|
|
# constants
|
|
#
|
|
|
|
GGUF_MAGIC = 0x46554747 # "GGUF"
|
|
GGUF_VERSION = 3
|
|
GGUF_DEFAULT_ALIGNMENT = 32
|
|
|
|
#
|
|
# metadata keys
|
|
#
|
|
|
|
|
|
class Keys:
|
|
class General:
|
|
ARCHITECTURE = "general.architecture"
|
|
QUANTIZATION_VERSION = "general.quantization_version"
|
|
ALIGNMENT = "general.alignment"
|
|
NAME = "general.name"
|
|
AUTHOR = "general.author"
|
|
URL = "general.url"
|
|
DESCRIPTION = "general.description"
|
|
LICENSE = "general.license"
|
|
SOURCE_URL = "general.source.url"
|
|
SOURCE_HF_REPO = "general.source.huggingface.repository"
|
|
FILE_TYPE = "general.file_type"
|
|
|
|
class LLM:
|
|
VOCAB_SIZE = "{arch}.vocab_size"
|
|
CONTEXT_LENGTH = "{arch}.context_length"
|
|
EMBEDDING_LENGTH = "{arch}.embedding_length"
|
|
BLOCK_COUNT = "{arch}.block_count"
|
|
FEED_FORWARD_LENGTH = "{arch}.feed_forward_length"
|
|
USE_PARALLEL_RESIDUAL = "{arch}.use_parallel_residual"
|
|
TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout"
|
|
EXPERT_COUNT = "{arch}.expert_count"
|
|
EXPERT_USED_COUNT = "{arch}.expert_used_count"
|
|
POOLING_TYPE = "{arch}.pooling_type"
|
|
LOGIT_SCALE = "{arch}.logit_scale"
|
|
|
|
class Attention:
|
|
HEAD_COUNT = "{arch}.attention.head_count"
|
|
HEAD_COUNT_KV = "{arch}.attention.head_count_kv"
|
|
MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias"
|
|
CLAMP_KQV = "{arch}.attention.clamp_kqv"
|
|
KEY_LENGTH = "{arch}.attention.key_length"
|
|
VALUE_LENGTH = "{arch}.attention.value_length"
|
|
LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon"
|
|
LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon"
|
|
CAUSAL = "{arch}.attention.causal"
|
|
|
|
class Rope:
|
|
DIMENSION_COUNT = "{arch}.rope.dimension_count"
|
|
FREQ_BASE = "{arch}.rope.freq_base"
|
|
SCALING_TYPE = "{arch}.rope.scaling.type"
|
|
SCALING_FACTOR = "{arch}.rope.scaling.factor"
|
|
SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length"
|
|
SCALING_FINETUNED = "{arch}.rope.scaling.finetuned"
|
|
|
|
class SSM:
|
|
CONV_KERNEL = "{arch}.ssm.conv_kernel"
|
|
INNER_SIZE = "{arch}.ssm.inner_size"
|
|
STATE_SIZE = "{arch}.ssm.state_size"
|
|
TIME_STEP_RANK = "{arch}.ssm.time_step_rank"
|
|
|
|
class Tokenizer:
|
|
MODEL = "tokenizer.ggml.model"
|
|
LIST = "tokenizer.ggml.tokens"
|
|
TOKEN_TYPE = "tokenizer.ggml.token_type"
|
|
TOKEN_TYPE_COUNT = "tokenizer.ggml.token_type_count" # for BERT-style token types
|
|
SCORES = "tokenizer.ggml.scores"
|
|
MERGES = "tokenizer.ggml.merges"
|
|
BOS_ID = "tokenizer.ggml.bos_token_id"
|
|
EOS_ID = "tokenizer.ggml.eos_token_id"
|
|
UNK_ID = "tokenizer.ggml.unknown_token_id"
|
|
SEP_ID = "tokenizer.ggml.seperator_token_id"
|
|
PAD_ID = "tokenizer.ggml.padding_token_id"
|
|
CLS_ID = "tokenizer.ggml.cls_token_id"
|
|
MASK_ID = "tokenizer.ggml.mask_token_id"
|
|
ADD_BOS = "tokenizer.ggml.add_bos_token"
|
|
ADD_EOS = "tokenizer.ggml.add_eos_token"
|
|
ADD_PREFIX = "tokenizer.ggml.add_space_prefix"
|
|
HF_JSON = "tokenizer.huggingface.json"
|
|
RWKV = "tokenizer.rwkv.world"
|
|
CHAT_TEMPLATE = "tokenizer.chat_template"
|
|
|
|
|
|
#
|
|
# recommended mapping of model tensor names for storage in gguf
|
|
#
|
|
|
|
|
|
class MODEL_ARCH(IntEnum):
|
|
LLAMA = auto()
|
|
FALCON = auto()
|
|
BAICHUAN = auto()
|
|
GROK = auto()
|
|
GPT2 = auto()
|
|
GPTJ = auto()
|
|
GPTNEOX = auto()
|
|
MPT = auto()
|
|
STARCODER = auto()
|
|
PERSIMMON = auto()
|
|
REFACT = auto()
|
|
BERT = auto()
|
|
NOMIC_BERT = auto()
|
|
BLOOM = auto()
|
|
STABLELM = auto()
|
|
QWEN = auto()
|
|
QWEN2 = auto()
|
|
PHI2 = auto()
|
|
PLAMO = auto()
|
|
CODESHELL = auto()
|
|
ORION = auto()
|
|
INTERNLM2 = auto()
|
|
MINICPM = auto()
|
|
GEMMA = auto()
|
|
STARCODER2 = auto()
|
|
MAMBA = auto()
|
|
COMMAND_R = auto()
|
|
|
|
|
|
class MODEL_TENSOR(IntEnum):
|
|
TOKEN_EMBD = auto()
|
|
TOKEN_EMBD_NORM = auto()
|
|
TOKEN_TYPES = auto()
|
|
POS_EMBD = auto()
|
|
OUTPUT = auto()
|
|
OUTPUT_NORM = auto()
|
|
ROPE_FREQS = auto()
|
|
ATTN_Q = auto()
|
|
ATTN_K = auto()
|
|
ATTN_V = auto()
|
|
ATTN_QKV = auto()
|
|
ATTN_OUT = auto()
|
|
ATTN_NORM = auto()
|
|
ATTN_NORM_2 = auto()
|
|
ATTN_OUT_NORM = auto()
|
|
ATTN_ROT_EMBD = auto()
|
|
FFN_GATE_INP = auto()
|
|
FFN_NORM = auto()
|
|
FFN_GATE = auto()
|
|
FFN_DOWN = auto()
|
|
FFN_UP = auto()
|
|
FFN_ACT = auto()
|
|
FFN_GATE_EXP = auto()
|
|
FFN_DOWN_EXP = auto()
|
|
FFN_UP_EXP = auto()
|
|
ATTN_Q_NORM = auto()
|
|
ATTN_K_NORM = auto()
|
|
LAYER_OUT_NORM = auto()
|
|
SSM_IN = auto()
|
|
SSM_CONV1D = auto()
|
|
SSM_X = auto()
|
|
SSM_DT = auto()
|
|
SSM_A = auto()
|
|
SSM_D = auto()
|
|
SSM_OUT = auto()
|
|
|
|
|
|
MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
|
MODEL_ARCH.LLAMA: "llama",
|
|
MODEL_ARCH.FALCON: "falcon",
|
|
MODEL_ARCH.BAICHUAN: "baichuan",
|
|
MODEL_ARCH.GROK: "grok",
|
|
MODEL_ARCH.GPT2: "gpt2",
|
|
MODEL_ARCH.GPTJ: "gptj",
|
|
MODEL_ARCH.GPTNEOX: "gptneox",
|
|
MODEL_ARCH.MPT: "mpt",
|
|
MODEL_ARCH.STARCODER: "starcoder",
|
|
MODEL_ARCH.PERSIMMON: "persimmon",
|
|
MODEL_ARCH.REFACT: "refact",
|
|
MODEL_ARCH.BERT: "bert",
|
|
MODEL_ARCH.NOMIC_BERT: "nomic-bert",
|
|
MODEL_ARCH.BLOOM: "bloom",
|
|
MODEL_ARCH.STABLELM: "stablelm",
|
|
MODEL_ARCH.QWEN: "qwen",
|
|
MODEL_ARCH.QWEN2: "qwen2",
|
|
MODEL_ARCH.PHI2: "phi2",
|
|
MODEL_ARCH.PLAMO: "plamo",
|
|
MODEL_ARCH.CODESHELL: "codeshell",
|
|
MODEL_ARCH.ORION: "orion",
|
|
MODEL_ARCH.INTERNLM2: "internlm2",
|
|
MODEL_ARCH.MINICPM: "minicpm",
|
|
MODEL_ARCH.GEMMA: "gemma",
|
|
MODEL_ARCH.STARCODER2: "starcoder2",
|
|
MODEL_ARCH.MAMBA: "mamba",
|
|
MODEL_ARCH.COMMAND_R: "command-r",
|
|
}
|
|
|
|
TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
|
MODEL_TENSOR.TOKEN_EMBD: "token_embd",
|
|
MODEL_TENSOR.TOKEN_EMBD_NORM: "token_embd_norm",
|
|
MODEL_TENSOR.TOKEN_TYPES: "token_types",
|
|
MODEL_TENSOR.POS_EMBD: "position_embd",
|
|
MODEL_TENSOR.OUTPUT_NORM: "output_norm",
|
|
MODEL_TENSOR.OUTPUT: "output",
|
|
MODEL_TENSOR.ROPE_FREQS: "rope_freqs",
|
|
MODEL_TENSOR.ATTN_NORM: "blk.{bid}.attn_norm",
|
|
MODEL_TENSOR.ATTN_NORM_2: "blk.{bid}.attn_norm_2",
|
|
MODEL_TENSOR.ATTN_QKV: "blk.{bid}.attn_qkv",
|
|
MODEL_TENSOR.ATTN_Q: "blk.{bid}.attn_q",
|
|
MODEL_TENSOR.ATTN_K: "blk.{bid}.attn_k",
|
|
MODEL_TENSOR.ATTN_V: "blk.{bid}.attn_v",
|
|
MODEL_TENSOR.ATTN_OUT: "blk.{bid}.attn_output",
|
|
MODEL_TENSOR.ATTN_ROT_EMBD: "blk.{bid}.attn_rot_embd",
|
|
MODEL_TENSOR.ATTN_Q_NORM: "blk.{bid}.attn_q_norm",
|
|
MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm",
|
|
MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm",
|
|
MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp",
|
|
MODEL_TENSOR.FFN_NORM: "blk.{bid}.ffn_norm",
|
|
MODEL_TENSOR.FFN_GATE: "blk.{bid}.ffn_gate",
|
|
MODEL_TENSOR.FFN_DOWN: "blk.{bid}.ffn_down",
|
|
MODEL_TENSOR.FFN_UP: "blk.{bid}.ffn_up",
|
|
MODEL_TENSOR.FFN_ACT: "blk.{bid}.ffn",
|
|
MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate.{xid}",
|
|
MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down.{xid}",
|
|
MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up.{xid}",
|
|
MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm",
|
|
MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in",
|
|
MODEL_TENSOR.SSM_CONV1D: "blk.{bid}.ssm_conv1d",
|
|
MODEL_TENSOR.SSM_X: "blk.{bid}.ssm_x",
|
|
MODEL_TENSOR.SSM_DT: "blk.{bid}.ssm_dt",
|
|
MODEL_TENSOR.SSM_A: "blk.{bid}.ssm_a",
|
|
MODEL_TENSOR.SSM_D: "blk.{bid}.ssm_d",
|
|
MODEL_TENSOR.SSM_OUT: "blk.{bid}.ssm_out",
|
|
}
|
|
|
|
MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
|
MODEL_ARCH.LLAMA: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_GATE_INP,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.FFN_GATE_EXP,
|
|
MODEL_TENSOR.FFN_DOWN_EXP,
|
|
MODEL_TENSOR.FFN_UP_EXP,
|
|
],
|
|
MODEL_ARCH.GROK: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.ATTN_OUT_NORM,
|
|
MODEL_TENSOR.FFN_GATE_INP,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.FFN_GATE_EXP,
|
|
MODEL_TENSOR.FFN_DOWN_EXP,
|
|
MODEL_TENSOR.FFN_UP_EXP,
|
|
MODEL_TENSOR.LAYER_OUT_NORM,
|
|
],
|
|
MODEL_ARCH.GPTNEOX: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.FALCON: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_NORM_2,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.BAICHUAN: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.STARCODER: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.POS_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.BERT: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.TOKEN_EMBD_NORM,
|
|
MODEL_TENSOR.TOKEN_TYPES,
|
|
MODEL_TENSOR.POS_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.ATTN_OUT_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.LAYER_OUT_NORM,
|
|
],
|
|
MODEL_ARCH.NOMIC_BERT: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.TOKEN_EMBD_NORM,
|
|
MODEL_TENSOR.TOKEN_TYPES,
|
|
MODEL_TENSOR.POS_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.ATTN_OUT_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.LAYER_OUT_NORM,
|
|
],
|
|
MODEL_ARCH.MPT: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.FFN_ACT,
|
|
],
|
|
MODEL_ARCH.GPTJ: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.PERSIMMON: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.ATTN_Q_NORM,
|
|
MODEL_TENSOR.ATTN_K_NORM,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
MODEL_ARCH.REFACT: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.BLOOM: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.TOKEN_EMBD_NORM,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.STABLELM: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.QWEN: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.QWEN2: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.PLAMO: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.GPT2: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.POS_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.PHI2: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.CODESHELL: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.POS_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_QKV,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.ORION: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.INTERNLM2: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.MINICPM: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_GATE_INP,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.FFN_GATE_EXP,
|
|
MODEL_TENSOR.FFN_DOWN_EXP,
|
|
MODEL_TENSOR.FFN_UP_EXP,
|
|
],
|
|
MODEL_ARCH.GEMMA: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
],
|
|
MODEL_ARCH.STARCODER2: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
MODEL_TENSOR.FFN_NORM,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
MODEL_ARCH.MAMBA: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.OUTPUT,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.SSM_IN,
|
|
MODEL_TENSOR.SSM_CONV1D,
|
|
MODEL_TENSOR.SSM_X,
|
|
MODEL_TENSOR.SSM_DT,
|
|
MODEL_TENSOR.SSM_A,
|
|
MODEL_TENSOR.SSM_D,
|
|
MODEL_TENSOR.SSM_OUT,
|
|
],
|
|
MODEL_ARCH.COMMAND_R: [
|
|
MODEL_TENSOR.TOKEN_EMBD,
|
|
MODEL_TENSOR.OUTPUT_NORM,
|
|
MODEL_TENSOR.ATTN_NORM,
|
|
MODEL_TENSOR.ATTN_Q,
|
|
MODEL_TENSOR.ATTN_K,
|
|
MODEL_TENSOR.ATTN_V,
|
|
MODEL_TENSOR.ATTN_OUT,
|
|
MODEL_TENSOR.FFN_GATE,
|
|
MODEL_TENSOR.FFN_DOWN,
|
|
MODEL_TENSOR.FFN_UP,
|
|
],
|
|
# TODO
|
|
}
|
|
|
|
# tensors that will not be serialized
|
|
MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
|
MODEL_ARCH.LLAMA: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
MODEL_ARCH.BAICHUAN: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
MODEL_ARCH.PERSIMMON: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
],
|
|
MODEL_ARCH.QWEN: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
MODEL_ARCH.CODESHELL: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
MODEL_ARCH.ORION: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
MODEL_ARCH.STARCODER2: [
|
|
MODEL_TENSOR.ROPE_FREQS,
|
|
MODEL_TENSOR.ATTN_ROT_EMBD,
|
|
],
|
|
}
|
|
|
|
#
|
|
# types
|
|
#
|
|
|
|
|
|
class TokenType(IntEnum):
|
|
NORMAL = 1
|
|
UNKNOWN = 2
|
|
CONTROL = 3
|
|
USER_DEFINED = 4
|
|
UNUSED = 5
|
|
BYTE = 6
|
|
|
|
|
|
class RopeScalingType(Enum):
|
|
NONE = 'none'
|
|
LINEAR = 'linear'
|
|
YARN = 'yarn'
|
|
|
|
|
|
class PoolingType(IntEnum):
|
|
NONE = 0
|
|
MEAN = 1
|
|
CLS = 2
|
|
|
|
|
|
class GGMLQuantizationType(IntEnum):
|
|
F32 = 0
|
|
F16 = 1
|
|
Q4_0 = 2
|
|
Q4_1 = 3
|
|
Q5_0 = 6
|
|
Q5_1 = 7
|
|
Q8_0 = 8
|
|
Q8_1 = 9
|
|
Q2_K = 10
|
|
Q3_K = 11
|
|
Q4_K = 12
|
|
Q5_K = 13
|
|
Q6_K = 14
|
|
Q8_K = 15
|
|
IQ2_XXS = 16
|
|
IQ2_XS = 17
|
|
IQ3_XXS = 18
|
|
IQ1_S = 19
|
|
IQ4_NL = 20
|
|
IQ3_S = 21
|
|
IQ2_S = 22
|
|
IQ4_XS = 23
|
|
I8 = 24
|
|
I16 = 25
|
|
I32 = 26
|
|
I64 = 27
|
|
F64 = 28
|
|
IQ1_M = 29
|
|
|
|
|
|
class GGUFEndian(IntEnum):
|
|
LITTLE = 0
|
|
BIG = 1
|
|
|
|
|
|
class GGUFValueType(IntEnum):
|
|
UINT8 = 0
|
|
INT8 = 1
|
|
UINT16 = 2
|
|
INT16 = 3
|
|
UINT32 = 4
|
|
INT32 = 5
|
|
FLOAT32 = 6
|
|
BOOL = 7
|
|
STRING = 8
|
|
ARRAY = 9
|
|
UINT64 = 10
|
|
INT64 = 11
|
|
FLOAT64 = 12
|
|
|
|
@staticmethod
|
|
def get_type(val: Any) -> GGUFValueType:
|
|
if isinstance(val, (str, bytes, bytearray)):
|
|
return GGUFValueType.STRING
|
|
elif isinstance(val, list):
|
|
return GGUFValueType.ARRAY
|
|
elif isinstance(val, float):
|
|
return GGUFValueType.FLOAT32
|
|
elif isinstance(val, bool):
|
|
return GGUFValueType.BOOL
|
|
elif isinstance(val, int):
|
|
return GGUFValueType.INT32
|
|
# TODO: need help with 64-bit types in Python
|
|
else:
|
|
print("Unknown type:", type(val))
|
|
sys.exit()
|
|
|
|
|
|
# Note: Does not support GGML_QKK_64
|
|
QK_K = 256
|
|
# Items here are (block size, type size)
|
|
GGML_QUANT_SIZES = {
|
|
GGMLQuantizationType.F32: (1, 4),
|
|
GGMLQuantizationType.F16: (1, 2),
|
|
GGMLQuantizationType.Q4_0: (32, 2 + 16),
|
|
GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16),
|
|
GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16),
|
|
GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16),
|
|
GGMLQuantizationType.Q8_0: (32, 2 + 32),
|
|
GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32),
|
|
GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4),
|
|
GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12),
|
|
GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12),
|
|
GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),
|
|
GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),
|
|
GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8),
|
|
GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4),
|
|
GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32),
|
|
GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8),
|
|
GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16),
|
|
GGMLQuantizationType.IQ4_NL: (32, 2 + 16),
|
|
GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4),
|
|
GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16),
|
|
GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64),
|
|
GGMLQuantizationType.I8: (1, 1),
|
|
GGMLQuantizationType.I16: (1, 2),
|
|
GGMLQuantizationType.I32: (1, 4),
|
|
GGMLQuantizationType.I64: (1, 8),
|
|
GGMLQuantizationType.F64: (1, 8),
|
|
}
|
|
|
|
|
|
# Aliases for backward compatibility.
|
|
|
|
# general
|
|
KEY_GENERAL_ARCHITECTURE = Keys.General.ARCHITECTURE
|
|
KEY_GENERAL_QUANTIZATION_VERSION = Keys.General.QUANTIZATION_VERSION
|
|
KEY_GENERAL_ALIGNMENT = Keys.General.ALIGNMENT
|
|
KEY_GENERAL_NAME = Keys.General.NAME
|
|
KEY_GENERAL_AUTHOR = Keys.General.AUTHOR
|
|
KEY_GENERAL_URL = Keys.General.URL
|
|
KEY_GENERAL_DESCRIPTION = Keys.General.DESCRIPTION
|
|
KEY_GENERAL_LICENSE = Keys.General.LICENSE
|
|
KEY_GENERAL_SOURCE_URL = Keys.General.SOURCE_URL
|
|
KEY_GENERAL_SOURCE_HF_REPO = Keys.General.SOURCE_HF_REPO
|
|
KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE
|
|
|
|
# LLM
|
|
KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE
|
|
KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH
|
|
KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH
|
|
KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT
|
|
KEY_FEED_FORWARD_LENGTH = Keys.LLM.FEED_FORWARD_LENGTH
|
|
KEY_USE_PARALLEL_RESIDUAL = Keys.LLM.USE_PARALLEL_RESIDUAL
|
|
KEY_TENSOR_DATA_LAYOUT = Keys.LLM.TENSOR_DATA_LAYOUT
|
|
|
|
# attention
|
|
KEY_ATTENTION_HEAD_COUNT = Keys.Attention.HEAD_COUNT
|
|
KEY_ATTENTION_HEAD_COUNT_KV = Keys.Attention.HEAD_COUNT_KV
|
|
KEY_ATTENTION_MAX_ALIBI_BIAS = Keys.Attention.MAX_ALIBI_BIAS
|
|
KEY_ATTENTION_CLAMP_KQV = Keys.Attention.CLAMP_KQV
|
|
KEY_ATTENTION_LAYERNORM_EPS = Keys.Attention.LAYERNORM_EPS
|
|
KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS
|
|
|
|
# RoPE
|
|
KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT
|
|
KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE
|
|
KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE
|
|
KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR
|
|
KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN
|
|
KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED
|
|
|
|
# SSM
|
|
KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL
|
|
KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE
|
|
KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE
|
|
KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK
|
|
|
|
# tokenization
|
|
KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL
|
|
KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST
|
|
KEY_TOKENIZER_TOKEN_TYPE = Keys.Tokenizer.TOKEN_TYPE
|
|
KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES
|
|
KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES
|
|
KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID
|
|
KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID
|
|
KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID
|
|
KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID
|
|
KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID
|
|
KEY_TOKENIZER_CLS_ID = Keys.Tokenizer.CLS_ID
|
|
KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID
|
|
KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON
|
|
KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV
|