root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-30 21:34:36 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

py : pad with unknown tokens when data is missing

Browse Source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-01-16 14:03:57 +02:00
parent 9b464b4e81
commit a1372737e0
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
convert.py
View File

@ -1098,6 +1098,15 @@ class OutputFile:
scores.append(score) scores.append(score)
toktypes.append(toktype) toktypes.append(toktype)
# pad with unknown tokens and print warnings
# ref: https://github.com/ggerganov/llama.cpp/issues/4958
if len(tokens) < vocab.vocab_size:
for i in range(len(tokens), vocab.vocab_size):
tokens.append(f"<unk{i}>".encode("utf-8"))
scores.append(-1000.0)
toktypes.append(gguf.TokenType.UNKNOWN)
print(f"Warning: token {i} not found in vocab - padding with {tokens[-1]}")
return tokens, scores, toktypes return tokens, scores, toktypes
def add_meta_vocab(self, vocab: Vocab) -> None: def add_meta_vocab(self, vocab: Vocab) -> None: