From d2819d5577b35507be83d0c3f4d2d3c0ab1488ca Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Fri, 16 Feb 2024 15:14:40 +0200
Subject: [PATCH] scripts : add helpers script for bench comparing commits
 (#5521)

* scripts : add helpers script for bench comparing commits

* scripts : detect CUDA

* set flags after checking the command line

* fix make flags

---------

Co-authored-by: slaren <slarengh@gmail.com>
---
 scripts/compare-commits.sh | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100755 scripts/compare-commits.sh

diff --git a/scripts/compare-commits.sh b/scripts/compare-commits.sh
new file mode 100755
index 000000000..331c4b9ce
--- /dev/null
+++ b/scripts/compare-commits.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+    echo "usage: ./scripts/compare-commits.sh <commit1> <commit2> [additional llama-bench arguments]"
+    exit 1
+fi
+
+set -e
+set -x
+
+bench_args="${@:3}"
+
+rm -f llama-bench.sqlite
+
+backend="cpu"
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    backend="metal"
+elif command -v nvcc &> /dev/null; then
+    backend="cuda"
+fi
+
+make_opts=""
+
+if [[ "$backend" == "cuda" ]]; then
+    make_opts="LLAMA_CUBLAS=1"
+fi
+
+git checkout $1
+make clean && make -j32 $make_opts llama-bench
+./llama-bench -o sql $bench_args | tee /dev/tty | sqlite3 llama-bench.sqlite
+
+git checkout $2
+make clean && make -j32 $make_opts llama-bench
+./llama-bench -o sql $bench_args | tee /dev/tty | sqlite3 llama-bench.sqlite
+
+./scripts/compare-llama-bench.py -b $1 -c $2