root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-26 03:14:35 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

convert : use utf8 encoding (#7000)

Browse Source 
* convert : use utf8 encoding

* convert : update instructions and warning message
This commit is contained in:
Georgi Gerganov 2024-04-30 11:05:25 +03:00 committed by GitHub
parent 8843a98c2b
commit 952d03dbea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 18 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
convert-hf-to-gguf-update.py
View File

@ -128,7 +128,7 @@ for model in models:
print(f"chkhsh: {chkhsh}") print(f"chkhsh: {chkhsh}")
# print the "pre_tokenizer" content from the tokenizer.json # print the "pre_tokenizer" content from the tokenizer.json
with open(f"models/tokenizers/{name}/tokenizer.json", "r") as f: with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
cfg = json.load(f) cfg = json.load(f)
pre_tokenizer = cfg["pre_tokenizer"] pre_tokenizer = cfg["pre_tokenizer"]
print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4)) print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
@ -156,15 +156,19 @@ src_func += " print(f\"chkhsh: {chkhsh}\")\n"
src_func += "\n" src_func += "\n"
src_func += " res = None\n" src_func += " res = None\n"
src_func += "\n" src_func += "\n"
src_func += " # NOTE: if you get an error here, you need to add the model to the if-elif chain below\n" src_func += " # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script\n"
src_func += " # don't do this manually - use the convert-hf-to-gguf-update.py script!\n" src_func += " # or pull the latest version of the model from Huggingface\n"
src_func += " # don't edit the hashes manually!\n"
src_func += f"{src_ifs}\n" src_func += f"{src_ifs}\n"
src_func += " if res is None:\n" src_func += " if res is None:\n"
src_func += " print(\"\\n\")\n" src_func += " print(\"\\n\")\n"
src_func += " print(\"**************************************************************************************\")\n" src_func += " print(\"**************************************************************************************\")\n"
src_func += " print(\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n" src_func += " print(\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n"
src_func += " print(\"** This means that it was not added yet or you are using an older version.\")\n" src_func += " print(\"** There are 2 possible reasons for this:\")\n"
src_func += " print(\"** Check convert-hf-to-gguf-update.py and update it accordingly.\")\n" src_func += " print(\"** - the model has not been added to convert-hf-to-gguf-update.py yet\")\n"
src_func += " print(\"** - the pre-tokenization config has changed upstream\")\n"
src_func += " print(\"** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.\")\n"
src_func += " print(\"** ref: https://github.com/ggerganov/llama.cpp/pull/6920\")\n"
src_func += " print(\"**\")\n" src_func += " print(\"**\")\n"
src_func += " print(f\"** chkhsh: {chkhsh}\")\n" src_func += " print(f\"** chkhsh: {chkhsh}\")\n"
src_func += " print(\"**************************************************************************************\")\n" src_func += " print(\"**************************************************************************************\")\n"
@ -249,7 +253,7 @@ for model in models:
from transformers import AutoTokenizer from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}") tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
with open(f"models/ggml-vocab-{name}.gguf.inp", "w") as f: with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
for text in tests: for text in tests:
f.write(f"{text}") f.write(f"{text}")
f.write("\n__ggml_vocab_test__\n") f.write("\n__ggml_vocab_test__\n")

12
convert-hf-to-gguf.py
View File

@ -279,8 +279,9 @@ class Model(ABC):
res = None res = None
# NOTE: if you get an error here, you need to add the model to the if-elif chain below # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script
# don't do this manually - use the convert-hf-to-gguf-update.py script! # or pull the latest version of the model from Huggingface
# don't edit the hashes manually!
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
res = "llama-bpe" res = "llama-bpe"
@ -310,8 +311,11 @@ class Model(ABC):
print("\n") print("\n")
print("**************************************************************************************") print("**************************************************************************************")
print("** WARNING: The BPE pre-tokenizer was not recognized!") print("** WARNING: The BPE pre-tokenizer was not recognized!")
print("** This means that it was not added yet or you are using an older version.") print("** There are 2 possible reasons for this:")
print("** Check convert-hf-to-gguf-update.py and update it accordingly.") print("** - the model has not been added to convert-hf-to-gguf-update.py yet")
print("** - the pre-tokenization config has changed upstream")
print("** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.")
print("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")
print("**") print("**")
print(f"** chkhsh: {chkhsh}") print(f"** chkhsh: {chkhsh}")
print("**************************************************************************************") print("**************************************************************************************")