root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-28 12:24:35 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

convert-llama-7b-pth-to-gguf.py : add token types

Browse Source
This commit is contained in:
klosax 2023-08-14 22:10:50 +02:00 committed by GitHub
parent cedb4870c6
commit ab2cbd03ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
convert-llama-7b-pth-to-gguf.py
View File

@ -110,8 +110,9 @@ gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"])
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: List[str] = [] tokens: List[bytes] = []
scores: List[float] = [] scores: List[float] = []
toktypes: List[int] = []
if Path(dir_model + "/tokenizer.model").is_file(): if Path(dir_model + "/tokenizer.model").is_file():
# vocab type sentencepiece # vocab type sentencepiece
@ -121,26 +122,31 @@ if Path(dir_model + "/tokenizer.model").is_file():
for i in range(tokenizer.vocab_size()): for i in range(tokenizer.vocab_size()):
text: bytes text: bytes
if tokenizer.is_unknown(i): score: float
text = " \u2047 ".encode("utf-8")
elif tokenizer.is_control(i):
text = b""
if tokenizer.is_byte(i):
piece = tokenizer.id_to_piece(i) piece = tokenizer.id_to_piece(i)
if len(piece) != 6: text = piece.encode("utf-8")
raise Exception(f"Invalid token: {piece}") score = tokenizer.get_score(i)
byte_value = int(piece[3:-1], 16)
text = struct.pack("B", byte_value) toktype = 1 # defualt to normal token type
else: if tokenizer.is_unknown(i): toktype = 2
text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") if tokenizer.is_control(i): toktype = 3
score: float = tokenizer.get_score(i)
# TODO: How to determinate if a token is user defined?
# ref: https://github.com/google/sentencepiece/blob/master/src/sentencepiece_model.proto
# if tokenizer.is_user_defined(i): toktype = 4
if tokenizer.is_unused(i): toktype = 5
if tokenizer.is_byte(i): toktype = 6
tokens.append(text) tokens.append(text)
scores.append(score) scores.append(score)
toktypes.append(toktype)
gguf_writer.add_tokenizer_model("llama") gguf_writer.add_tokenizer_model("llama")
gguf_writer.add_token_list(tokens) gguf_writer.add_token_list(tokens)
gguf_writer.add_token_scores(scores) gguf_writer.add_token_scores(scores)
gguf_writer.add_token_types(toktypes)
if Path(dir_model + "/tokenizer.json").is_file(): if Path(dir_model + "/tokenizer.json").is_file():
with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: