root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-11-11 13:30:35 +00:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Fix convert-llama-ggmlv3-to-gguf.py vocab conversion (#2698)

Browse Source 
When converting without metadata, the hex value for bytes entries weren't 0 padded to 2 digits.
This commit is contained in:
Kerfuffle 2023-08-21 18:01:34 -06:00 committed by GitHub
parent c8dba409e6
commit 930523c8e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
convert-llama-ggmlv3-to-gguf.py
View File

@ -236,8 +236,7 @@ class GGMLToGGUF:
if len(vbytes) == 0: if len(vbytes) == 0:
tt = 3 # Control tt = 3 # Control
elif tokid >= 3 and tokid <= 258 and len(vbytes) == 1: elif tokid >= 3 and tokid <= 258 and len(vbytes) == 1:
hv = hex(vbytes[0])[2:].upper() vbytes = bytes(f'<0x{vbytes[0]:02X}>', encoding = 'UTF-8')
vbytes = bytes(f'<0x{hv}>', encoding = 'UTF-8')
tt = 6 # Byte tt = 6 # Byte
else: else:
vbytes = vbytes.replace(b' ', b'\xe2\x96\x81') vbytes = vbytes.replace(b' ', b'\xe2\x96\x81')