root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-27 03:44:35 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

gguf : write tokenizer data

Browse Source
This commit is contained in:
M. Yusuf Sarıgöz 2023-07-29 13:30:22 +03:00
parent 8a76dd8a85
commit cc3dd7f042
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
gguf.py
View File

@ -267,6 +267,15 @@ class GGUFWriter:
def write_rope_scale(self, llm: str, value: float):
self.write_float32(constants.KEY_ROPE_SCALE.format(llm=llm), value)
def write_tokenizer_model(self, model: str):
self.write_string(constants.KEY_TOKENIZER_MODEL, model)
def write_token_list(self, tokens: List[str]):
self.write_array(constants.KEY_TOKENIZER_LIST, tokens)
def write_token_scores(self, scores: List[float]):
self.write_array(constants.KEY_TOKENIZER_SCORES, scores)
# Example usage:
if __name__ == "__main__":