root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-11-11 21:39:52 +00:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

convert-hf : Fix the encoding in the convert-hf-to-gguf-update.py (#8040)

Browse Source
This commit is contained in:
Hamdoud Hakem 2024-06-20 20:59:59 +01:00 committed by GitHub
parent abd894ad96
commit 17b291a6a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
convert-hf-to-gguf-update.py
View File

@ -214,7 +214,7 @@ src_func = f"""
""" """
convert_py_pth = pathlib.Path("convert-hf-to-gguf.py") convert_py_pth = pathlib.Path("convert-hf-to-gguf.py")
convert_py = convert_py_pth.read_text() convert_py = convert_py_pth.read_text(encoding="utf-8")
convert_py = re.sub( convert_py = re.sub(
r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)", r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)",
lambda m: m.group(1) + src_func + m.group(3), lambda m: m.group(1) + src_func + m.group(3),
@ -222,7 +222,7 @@ convert_py = re.sub(
flags=re.DOTALL | re.MULTILINE, flags=re.DOTALL | re.MULTILINE,
) )
convert_py_pth.write_text(convert_py) convert_py_pth.write_text(convert_py, encoding="utf-8")
logger.info("+++ convert-hf-to-gguf.py was updated") logger.info("+++ convert-hf-to-gguf.py was updated")