root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-30 21:34:36 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

py : fix missing added_tokens_dict for SPM vocab

Browse Source
This commit is contained in:
Georgi Gerganov 2024-01-16 13:38:54 +02:00
parent a0b3ac8c48
commit 9b464b4e81
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
convert.py
View File

@ -466,6 +466,7 @@ class SentencePieceVocab: # LlaMa
) )
# Token pieces that were added to the base vocabulary. # Token pieces that were added to the base vocabulary.
self.added_tokens_dict = added_tokens
self.added_tokens_list = [new_tokens[id] for id in actual_new_ids] self.added_tokens_list = [new_tokens[id] for id in actual_new_ids]
self.vocab_size_base = vocab_size self.vocab_size_base = vocab_size
self.vocab_size = self.vocab_size_base + len(self.added_tokens_list) self.vocab_size = self.vocab_size_base + len(self.added_tokens_list)