mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-09-22 21:16:20 +00:00
a2ac89d6ef
* convert.py: add python logging instead of print() * convert.py: verbose flag takes priority over dump flag log suppression * convert.py: named instance logging * convert.py: use explicit logger id string * convert.py: convert extra print() to named logger * convert.py: sys.stderr.write --> logger.error * *.py: Convert all python scripts to use logging module * requirements.txt: remove extra line * flake8: update flake8 ignore and exclude to match ci settings * gh-actions: add flake8-no-print to flake8 lint step * pre-commit: add flake8-no-print to flake8 and also update pre-commit version * convert-hf-to-gguf.py: print() to logger conversion * *.py: logging basiconfig refactor to use conditional expression * *.py: removed commented out logging * fixup! *.py: logging basiconfig refactor to use conditional expression * constant.py: logger.error then exit should be a raise exception instead * *.py: Convert logger error and sys.exit() into a raise exception (for atypical error) * gguf-convert-endian.py: refactor convert_byteorder() to use tqdm progressbar * verify-checksum-model.py: This is the result of the program, it should be printed to stdout. * compare-llama-bench.py: add blank line for readability during missing repo response * reader.py: read_gguf_file() use print() over logging * convert.py: warning goes to stderr and won't hurt the dump output * gguf-dump.py: dump_metadata() should print to stdout * convert-hf-to-gguf.py: print --> logger.debug or ValueError() * verify-checksum-models.py: use print() for printing table * *.py: refactor logging.basicConfig() * gguf-py/gguf/*.py: use __name__ as logger name Since they will be imported and not run directly. * python-lint.yml: use .flake8 file instead * constants.py: logger no longer required * convert-hf-to-gguf.py: add additional logging * convert-hf-to-gguf.py: print() --> logger * *.py: fix flake8 warnings * revert changes to convert-hf-to-gguf.py for get_name() * convert-hf-to-gguf-update.py: use triple quoted f-string instead * *.py: accidentally corrected the wrong line * *.py: add compilade warning suggestions and style fixes
85 lines
2.4 KiB
Python
Executable File
85 lines
2.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import logging
|
|
import os
|
|
import hashlib
|
|
|
|
logger = logging.getLogger("verify-checksum-models")
|
|
|
|
|
|
def sha256sum(file):
|
|
block_size = 16 * 1024 * 1024 # 16 MB block size
|
|
b = bytearray(block_size)
|
|
file_hash = hashlib.sha256()
|
|
mv = memoryview(b)
|
|
with open(file, 'rb', buffering=0) as f:
|
|
while True:
|
|
n = f.readinto(mv)
|
|
if not n:
|
|
break
|
|
file_hash.update(mv[:n])
|
|
|
|
return file_hash.hexdigest()
|
|
|
|
|
|
# Define the path to the llama directory (parent folder of script directory)
|
|
llama_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
|
|
|
|
# Define the file with the list of hashes and filenames
|
|
hash_list_file = os.path.join(llama_path, "SHA256SUMS")
|
|
|
|
# Check if the hash list file exists
|
|
if not os.path.exists(hash_list_file):
|
|
logger.error(f"Hash list file not found: {hash_list_file}")
|
|
exit(1)
|
|
|
|
# Read the hash file content and split it into an array of lines
|
|
with open(hash_list_file, "r") as f:
|
|
hash_list = f.read().splitlines()
|
|
|
|
# Create an array to store the results
|
|
results = []
|
|
|
|
# Loop over each line in the hash list
|
|
for line in hash_list:
|
|
# Split the line into hash and filename
|
|
hash_value, filename = line.split(" ")
|
|
|
|
# Get the full path of the file by joining the llama path and the filename
|
|
file_path = os.path.join(llama_path, filename)
|
|
|
|
# Informing user of the progress of the integrity check
|
|
logger.info(f"Verifying the checksum of {file_path}")
|
|
|
|
# Check if the file exists
|
|
if os.path.exists(file_path):
|
|
# Calculate the SHA256 checksum of the file using hashlib
|
|
file_hash = sha256sum(file_path)
|
|
|
|
# Compare the file hash with the expected hash
|
|
if file_hash == hash_value:
|
|
valid_checksum = "V"
|
|
file_missing = ""
|
|
else:
|
|
valid_checksum = ""
|
|
file_missing = ""
|
|
else:
|
|
valid_checksum = ""
|
|
file_missing = "X"
|
|
|
|
# Add the results to the array
|
|
results.append({
|
|
"filename": filename,
|
|
"valid checksum": valid_checksum,
|
|
"file missing": file_missing
|
|
})
|
|
|
|
|
|
# Print column headers for results table
|
|
print("filename".ljust(40) + "valid checksum".center(20) + "file missing".center(20)) # noqa: NP100
|
|
print("-" * 80) # noqa: NP100
|
|
|
|
# Output the results as a table
|
|
for r in results:
|
|
print(f"{r['filename']:40} {r['valid checksum']:^20} {r['file missing']:^20}") # noqa: NP100
|