From 15748844833b944b159c6b70e6af4d2d05f1108a Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 12 Nov 2024 22:26:56 -0800 Subject: [PATCH] Update test-tokenizer-random.py --- tests/test-tokenizer-random.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index 5b95e3f09..1ecc6a158 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -""" +# # Test libllama tokenizer == AutoTokenizer. # Brute force random words/text generation. - -Sample usage: - - python3 test-tokenizer-random.py ./models/ggml-vocab-llama-bpe.gguf ./models/tokenizers/llama-bpe -""" +# +#Sample usage: +# +# python3 test-tokenizer-random.py ./models/ggml-vocab-llama-bpe.gguf ./models/tokenizers/llama-bpe +# from __future__ import annotations #