diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md index 51a125e19..18407f688 100644 --- a/docs/debugging-tests.md +++ b/docs/debugging-tests.md @@ -1,6 +1,6 @@ # Debugging Tests Tips -## How to run & debug a specific test without anything else to keep the feedback loop short? +## How to run & execute or debug a specific test without anything else to keep the feedback loop short? There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. @@ -10,13 +10,27 @@ For example, running the following command will output an interactive list from It will then build & run in the debugger for you. +To just execute a test and get back a PASS or FAIL message run: + ```bash ./scripts/debug-test.sh test-tokenizer +``` + +To test in GDB use the `-g` flag to enable gdb test mode. + +```bash +./scripts/debug-test.sh -g test-tokenizer # Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows: >>> b main ``` +To speed up the testing loop, if you know your test number you can just run it similar to below: + +```bash +./scripts/debug-test.sh test 23 +``` + For further reference use `debug-test.sh -h` to print help.   @@ -41,7 +55,7 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON .. make -j ``` -#### Step 3.1: Identify Test Command for Debugging +#### Step 3: Find all tests available that matches REGEX The output of this command will give you the command & arguments needed to run GDB. @@ -69,11 +83,13 @@ Labels: main ... ``` -So for test #1 we can tell these two pieces of relevant information: +#### Step 4: Identify Test Command for Debugging + +So for test #1 above we can tell these two pieces of relevant information: * Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0` * Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf` -#### Step 3.2: Run GDB on test command +#### Step 5: Run GDB on test command Based on the ctest 'test command' report above we can then run a gdb session via this command below: diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh index 231a23d69..7b2b601a9 100755 --- a/scripts/debug-test.sh +++ b/scripts/debug-test.sh @@ -1,117 +1,203 @@ #!/bin/bash -test_suite=${1:-} -test_number=${2:-} PROG=${0##*/} build_dir="build-ci-debug" -if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Debug specific ctest program." - echo - echo "Options:" - echo " -h, --help Display this help and exit" - echo - echo "Arguments:" - echo " (Mandatory) Supply one regex to the script to filter tests" - echo " (test_number) (Optional) Test number to run a specific test" - echo - echo "Example:" - echo " $PROG test-tokenizer" - echo " $PROG test-tokenizer 3" - echo - exit 0 -fi +# Print Color Commands +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +normal=$(tput sgr0) -# Function to select and debug a test -function select_test() { - test_suite=${1:-test} - test_number=${2:-} - # Sanity Check If Tests Is Detected - printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" - tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) - if [ ${#tests[@]} -eq 0 ] - then - echo "No tests avaliable... check your compliation process..." - echo "Exiting." - exit 1 - fi +# Print Help Message +#################### - if [ -z $test_number ] - then - # List out avaliable tests - printf "Which test would you like to debug?\n" - id=0 - for s in "${tests[@]}" - do - echo "Test# ${id}" - echo " $s" - ((id++)) - done +print_full_help() { + cat << EOF +Usage: $PROG [OPTION]... (test_number) +Debug specific ctest program. - # Prompt user which test they wanted to run - printf "\nRun test#? " - read test_number - else - printf "\nUser Already Requested #${test_number}" - fi +Options: + -h, --help display this help and exit + -g run in gdb mode - # Start GDB with the requested test binary and arguments - printf "Debugging(GDB) test: ${tests[test_number]}\n" - # Change IFS (Internal Field Separator) - sIFS=$IFS - IFS=$'\n' +Arguments: + (Mandatory) Supply one regex to the script to filter tests + (test_number) (Optional) Test number to run a specific test - # Get test args - gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) - IFS=$sIFS - printf "Debug arguments: ${gdb_args[test_number]}\n\n" - - # Expand paths if needed - args=() - for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\"//') - do - args+=($(echo $x | sed -e 's/.*\/..\//..\//')) - done - - # Execute debugger - echo "gdb args: ${args[@]}" - gdb --args ${args[@]} +Example: + $PROG test-tokenizer + $PROG test-tokenizer 3 +EOF } +abort() { + echo "Error: $1" >&2 + cat << EOF >&2 +Usage: $PROG [OPTION]... (test_number) +Debug specific ctest program. +Refer to --help for full instructions. +EOF + exit 1 +} + + +# Dependency Sanity Check +######################### + +check_dependency() { + command -v "$1" >/dev/null 2>&1 || { + abort "$1 is required but not found. Please install it and try again." + } +} + +check_dependency ctest +check_dependency cmake + + # Step 0: Check the args -if [ -z "$test_suite" ] -then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Supply one regex to the script to filter tests," - echo "and optionally a test number to run a specific test." - echo "Use --help flag for full instructions" - exit 1 +######################## + +if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then + print_full_help >&2 + exit 0 fi +# Parse command-line options +gdb_mode=false +while getopts "g" opt; do + case $opt in + g) + gdb_mode=true + echo "gdb_mode Mode Enabled" + ;; + esac +done + +# Shift the option parameters +shift $((OPTIND - 1)) + +# Positionial Argument Processing : +if [ -z "${1}" ]; then + abort "Test regex is required" +else + test_suite=${1:-} +fi + +# Positionial Argument Processing : (test_number) +test_number=${2:-} + + # Step 1: Reset and Setup folder context +######################################## + ## Sanity check that we are actually in a git repo repo_root=$(git rev-parse --show-toplevel) if [ ! -d "$repo_root" ]; then - echo "Error: Not in a Git repository." - exit 1 + abort "Not in a Git repository." fi -## Reset folder to root context of git repo -pushd "$repo_root" || exit 1 +## Reset folder to root context of git repo and Create and enter build directory +pushd "$repo_root" +rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir" -## Create and enter build directory -rm -rf "$build_dir" && mkdir "$build_dir" || exit 1 # Step 2: Setup Build Environment and Compile Test Binaries -cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1 -pushd "$build_dir" && make -j || exit 1 +########################################################### -# Step 3: Debug the Test -select_test "$test_suite" "$test_number" +# Note: test-eval-callback requires -DLLAMA_CURL +cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_CURL=1 || abort "Failed to build enviroment" +pushd "$build_dir" +make -j || abort "Failed to compile" +popd > /dev/null || exit 1 -# Step 4: Return to the directory from which the user ran the command. -popd || exit 1 -popd || exit 1 -popd || exit 1 + +# Step 3: Find all tests available that matches REGEX +#################################################### + +# Ctest Gather Tests +# `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex) +# `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB. +# `-V` : Verbose Mode +printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" +pushd "$build_dir" +tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) +if [ ${#tests[@]} -eq 0 ]; then + abort "No tests avaliable... check your compliation process..." +fi +popd > /dev/null || exit 1 + + +# Step 4: Identify Test Command for Debugging +############################################# + +# Select test number +if [ -z $test_number ]; then + # List out avaliable tests + printf "Which test would you like to debug?\n" + id=0 + for s in "${tests[@]}" + do + echo "Test# ${id}" + echo " $s" + ((id++)) + done + + # Prompt user which test they wanted to run + printf "\nRun test#? " + read test_number + +else + printf "\nUser Already Requested #${test_number}\n" + +fi + +# Grab all tests commands +pushd "$build_dir" +sIFS=$IFS # Save Initial IFS (Internal Field Separator) +IFS=$'\n' # Change IFS (Internal Field Separator) (So we split ctest output by newline rather than by spaces) +test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) # Get test args +IFS=$sIFS # Reset IFS (Internal Field Separator) +popd > /dev/null || exit 1 + +# Grab specific test command +single_test_name="${tests[test_number]}" +single_test_command="${test_args[test_number]}" + + +# Step 5: Execute or GDB Debug +############################## + +printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n" +printf "${cyan}single_test_command: ${single_test_command}${normal}\n" + +if [ "$gdb_mode" = "true" ]; then + # Execute debugger + pushd "$repo_root" || exit 1 + eval "gdb --args ${single_test_command}" + popd > /dev/null || exit 1 + +else + # Execute Test + pushd "$repo_root" || exit 1 + eval "${single_test_command}" + exit_code=$? + popd > /dev/null || exit 1 + + # Print Result + printf "${blue}Ran Test #${test_number}: ${single_test_name}${normal}\n" + printf "${yellow}Command: ${single_test_command}${normal}\n" + if [ $exit_code -eq 0 ]; then + printf "${green}TEST PASS${normal}\n" + else + printf "${red}TEST FAIL${normal}\n" + fi + +fi + +# Return to the directory from which the user ran the command. +popd > /dev/null || exit 1