Fix convert-llama-ggmlv3-to-gguf.py vocab conversion (#2698)

When converting without metadata, the hex value for bytes entries weren't 0 padded to 2 digits.
This commit is contained in:
Kerfuffle 2023-08-21 18:01:34 -06:00 committed by GitHub
parent c8dba409e6
commit 930523c8e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -236,8 +236,7 @@ class GGMLToGGUF:
if len(vbytes) == 0:
tt = 3 # Control
elif tokid >= 3 and tokid <= 258 and len(vbytes) == 1:
hv = hex(vbytes[0])[2:].upper()
vbytes = bytes(f'<0x{hv}>', encoding = 'UTF-8')
vbytes = bytes(f'<0x{vbytes[0]:02X}>', encoding = 'UTF-8')
tt = 6 # Byte
else:
vbytes = vbytes.replace(b' ', b'\xe2\x96\x81')