2024-01-14 08:41:44 +00:00
|
|
|
cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories.
|
2023-03-21 00:37:16 +00:00
|
|
|
project("llama.cpp" C CXX)
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
include(CheckIncludeFileCXX)
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2023-03-21 15:29:41 +00:00
|
|
|
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
|
|
|
|
2023-03-13 19:22:15 +00:00
|
|
|
if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
|
|
|
|
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
|
|
|
|
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
|
|
|
|
endif()
|
|
|
|
|
2023-03-21 15:29:41 +00:00
|
|
|
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
|
|
|
|
2023-11-05 08:03:09 +00:00
|
|
|
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
|
2023-03-21 15:29:41 +00:00
|
|
|
set(LLAMA_STANDALONE ON)
|
|
|
|
|
|
|
|
# configure project version
|
|
|
|
# TODO
|
|
|
|
else()
|
|
|
|
set(LLAMA_STANDALONE OFF)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
if (EMSCRIPTEN)
|
|
|
|
set(BUILD_SHARED_LIBS_DEFAULT OFF)
|
|
|
|
|
|
|
|
option(LLAMA_WASM_SINGLE_FILE "llama: embed WASM inside the generated llama.js" ON)
|
|
|
|
else()
|
|
|
|
if (MINGW)
|
|
|
|
set(BUILD_SHARED_LIBS_DEFAULT OFF)
|
|
|
|
else()
|
|
|
|
set(BUILD_SHARED_LIBS_DEFAULT ON)
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
#
|
|
|
|
# Option list
|
|
|
|
#
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2023-09-04 19:26:24 +00:00
|
|
|
if (APPLE)
|
|
|
|
set(LLAMA_METAL_DEFAULT ON)
|
|
|
|
else()
|
|
|
|
set(LLAMA_METAL_DEFAULT OFF)
|
|
|
|
endif()
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
# general
|
2023-11-27 19:25:42 +00:00
|
|
|
option(BUILD_SHARED_LIBS "build shared libraries" OFF)
|
2023-05-25 21:07:29 +00:00
|
|
|
option(LLAMA_STATIC "llama: static link libraries" OFF)
|
2023-11-05 08:03:09 +00:00
|
|
|
option(LLAMA_NATIVE "llama: enable -march=native flag" ON)
|
2023-05-25 21:07:29 +00:00
|
|
|
option(LLAMA_LTO "llama: enable link time optimization" OFF)
|
2024-01-20 08:11:31 +00:00
|
|
|
option(LLAMA_CCACHE "llama: use ccache if available" ON)
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
# debug
|
2023-05-25 21:07:29 +00:00
|
|
|
option(LLAMA_ALL_WARNINGS "llama: enable all compiler warnings" ON)
|
|
|
|
option(LLAMA_ALL_WARNINGS_3RD_PARTY "llama: enable all compiler warnings in 3rd party libs" OFF)
|
|
|
|
option(LLAMA_GPROF "llama: enable gprof" OFF)
|
2023-03-21 00:37:16 +00:00
|
|
|
|
2024-02-17 21:03:14 +00:00
|
|
|
# build
|
|
|
|
option(LLAMA_FATAL_WARNINGS "llama: enable -Werror flag" OFF)
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
# sanitizers
|
2023-05-25 21:07:29 +00:00
|
|
|
option(LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" OFF)
|
|
|
|
option(LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" OFF)
|
|
|
|
option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF)
|
2023-03-21 00:37:16 +00:00
|
|
|
|
|
|
|
# instruction set specific
|
2023-10-03 16:53:15 +00:00
|
|
|
if (LLAMA_NATIVE)
|
|
|
|
set(INS_ENB OFF)
|
|
|
|
else()
|
|
|
|
set(INS_ENB ON)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
option(LLAMA_AVX "llama: enable AVX" ${INS_ENB})
|
|
|
|
option(LLAMA_AVX2 "llama: enable AVX2" ${INS_ENB})
|
|
|
|
option(LLAMA_AVX512 "llama: enable AVX512" OFF)
|
|
|
|
option(LLAMA_AVX512_VBMI "llama: enable AVX512-VBMI" OFF)
|
|
|
|
option(LLAMA_AVX512_VNNI "llama: enable AVX512-VNNI" OFF)
|
|
|
|
option(LLAMA_FMA "llama: enable FMA" ${INS_ENB})
|
2023-04-13 12:48:21 +00:00
|
|
|
# in MSVC F16C is implied with AVX2/AVX512
|
|
|
|
if (NOT MSVC)
|
2023-10-03 16:53:15 +00:00
|
|
|
option(LLAMA_F16C "llama: enable F16C" ${INS_ENB})
|
2023-04-13 12:48:21 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
|
2024-01-14 08:41:44 +00:00
|
|
|
if (WIN32)
|
2024-02-04 04:18:51 +00:00
|
|
|
set(LLAMA_WIN_VER "0x602" CACHE STRING "llama: Windows Version")
|
2024-01-14 08:41:44 +00:00
|
|
|
endif()
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
# 3rd party libs
|
2023-06-04 20:34:30 +00:00
|
|
|
option(LLAMA_ACCELERATE "llama: enable Accelerate framework" ON)
|
|
|
|
option(LLAMA_BLAS "llama: use BLAS" OFF)
|
2023-05-27 14:24:06 +00:00
|
|
|
set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
|
2023-07-29 21:04:44 +00:00
|
|
|
option(LLAMA_CUBLAS "llama: use CUDA" OFF)
|
2023-07-31 13:44:35 +00:00
|
|
|
#option(LLAMA_CUDA_CUBLAS "llama: use cuBLAS for prompt processing" OFF)
|
2023-07-05 12:19:42 +00:00
|
|
|
option(LLAMA_CUDA_FORCE_DMMV "llama: use dmmv instead of mmvq CUDA kernels" OFF)
|
2023-10-27 14:01:23 +00:00
|
|
|
option(LLAMA_CUDA_FORCE_MMQ "llama: use mmq kernels instead of cuBLAS" OFF)
|
2023-06-04 20:34:30 +00:00
|
|
|
set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
|
2023-07-05 12:19:42 +00:00
|
|
|
set(LLAMA_CUDA_MMV_Y "1" CACHE STRING "llama: y block size for mmv CUDA kernels")
|
2023-07-31 17:52:22 +00:00
|
|
|
option(LLAMA_CUDA_F16 "llama: use 16 bit floats for some calculations" OFF)
|
2023-06-16 17:08:44 +00:00
|
|
|
set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K")
|
2023-09-17 14:37:53 +00:00
|
|
|
set(LLAMA_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
|
|
|
|
"llama: max. batch size for using peer access")
|
2023-08-25 09:09:42 +00:00
|
|
|
option(LLAMA_HIPBLAS "llama: use hipBLAS" OFF)
|
2023-12-21 19:45:32 +00:00
|
|
|
option(LLAMA_HIP_UMA "llama: use HIP unified memory architecture" OFF)
|
2023-06-04 20:34:30 +00:00
|
|
|
option(LLAMA_CLBLAST "llama: use CLBlast" OFF)
|
ggml : add Vulkan backend (#2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
option(LLAMA_VULKAN "llama: use Vulkan" OFF)
|
2024-02-03 17:15:00 +00:00
|
|
|
option(LLAMA_VULKAN_CHECK_RESULTS "llama: run Vulkan op checks" OFF)
|
|
|
|
option(LLAMA_VULKAN_DEBUG "llama: enable Vulkan debug output" OFF)
|
|
|
|
option(LLAMA_VULKAN_VALIDATE "llama: enable Vulkan validation" OFF)
|
|
|
|
option(LLAMA_VULKAN_RUN_TESTS "llama: run Vulkan tests" OFF)
|
2023-09-04 19:26:24 +00:00
|
|
|
option(LLAMA_METAL "llama: use Metal" ${LLAMA_METAL_DEFAULT})
|
2023-09-05 22:21:10 +00:00
|
|
|
option(LLAMA_METAL_NDEBUG "llama: disable Metal debugging" OFF)
|
2024-01-02 08:57:44 +00:00
|
|
|
option(LLAMA_METAL_SHADER_DEBUG "llama: compile Metal with -fno-fast-math" OFF)
|
2024-01-29 20:50:50 +00:00
|
|
|
option(LLAMA_KOMPUTE "llama: use Kompute" OFF)
|
2023-07-10 15:49:56 +00:00
|
|
|
option(LLAMA_MPI "llama: use MPI" OFF)
|
k-quants : support for super-block size of 64 (#2001)
* k_quants: WIP super-blocks with 64 weights
* k_quants: WIP super-blocks with 64 weights
Q6_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q4_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q2_K scalar and AVX2 works. Q2_K is way too slow (it is actually slower
than the scalar implementation)
* k_quants: WIP super-blocks with 64 weights
Q3_K scalar and AVX2 works.
* k_quants: WIP super-blocks with 64 weights
Q5_K scalar and AVX2 works, and with that all
k_quants are done on AVX2 and scalar
* k_quants: WIP super-blocks with 64 weights
Q6_K working on CUDA. Cannot make it run quite as gast as
with super-blocks with 256 weigths: 8% slower on 4080,
20% slower on the 1660 (but there we fit 1 less layer on the
GPU because pf the larger model size), so some fraction of
these 20% is due to that,
* k_quants: WIP super-blocks with 64 weights
Q4_K working on CUDA. ~10% slower on GTX-1660,
16% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q2_K working on CUDA. ~3% slower on GTX-1660,
10% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q3_K working on CUDA.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on CUDA, and with this CUDA is done.
* k_quants: WIP super-blocks with 64 weights
Q6_K working on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Q4_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q2_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q3_K working on ARM_NEON, but quite a bit slower than 256 weights.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on ARM_NEON, but quite a bit slower than 256 weights.
With that, we have full support for ARM_NEON, although
performance is not quite there.
* k_quants: WIP super-blocks with 64 weights
Slightly more efficient Q3_K and Q5_K
* k_quants: WIP super-blocks with 64 weights
Another small improvement for Q3_K and Q5_K on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Yet another speedup for Q5_K on ARM_NEON.
We are now within 10% of the QK_K = 256 version.
* k_quants: WIP super-blocks with 64 weights
* We are able to pass preprocessor macros to the Metal
compiler
* Q6_K works and is actually slightly more efficient than
the QK_K = 256 version (25.2 ms vs 25.8 ms)
* k_quants: WIP super-blocks with 64 weights
Q4_K works on Metal and is actually slightly faster
than QK_K = 256 (21.95 ms vs 24.0 ms).
* k_quants: WIP super-blocks with 64 weights
Q2_K works on Metal and is very slightly faster
than QK_K = 256 (23.8 ms vs 24.2 ms).
* k_quants: WIP super-blocks with 64 weights
Q3_K works on Metal and is slightly faster
than QK_K = 256 (26.6 ms vs 28.3 ms).
* k_quants: WIP super-blocks with 64 weights
Q5_K works on Metal and is slightly faster
than QK_K = 256 (23.7 ms vs 26.3 ms).
* k_quants: call them _K, not _k, also on Metal
* k_quants: correctly define QK_K in llama.cpp
* Fixed bug in q4_K quantization added with the 64-block addition
* Simplify via lambda
* k_quants: swicth Q3_K to 4-bit scales when QK_K = 64
Otherwise there isn't much benefit from this
quantization type. There is some very slight loss
in accuracy, but we reduce size by ~7%.
E.g., for OpenLLaMA-3B, Q3_K_S perplexity is
8.6131 with 8-bit scales and 8.6352 with 4-bit,
while file size decreases from 1.53G to 1.44G.
* k_quants: switch Q4_K to 4-bit scales when QK_K = 64
Here the loss in accuracy is greater than for Q3_K,
but the Q4_K points still move further to the left on
the perplexity vs size curve.
* k_quants: forgot to add the Metal changes in last commit
* k_quants: change Q5_K to be type 0 when QK_K = 64
Still needs AVX2 implementation
* k_quants: AVX2 implementation for new 64-weight Q5_K
* k_quants: 10% faster ARM_NEON Q5_K dot product
* k_quants: fixed issue caused by merging with master
---------
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
2023-06-26 16:43:07 +00:00
|
|
|
option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF)
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
option(LLAMA_SYCL "llama: use SYCL" OFF)
|
|
|
|
option(LLAMA_SYCL_F16 "llama: use 16 bit floats for sycl calculations" OFF)
|
2024-02-16 17:05:56 +00:00
|
|
|
option(LLAMA_CPU_HBM "llama: use memkind for CPU HBM" OFF)
|
2023-05-25 21:07:29 +00:00
|
|
|
|
2023-12-07 20:26:54 +00:00
|
|
|
option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE})
|
|
|
|
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
|
|
|
option(LLAMA_BUILD_SERVER "llama: build server example" ON)
|
2023-03-21 15:29:41 +00:00
|
|
|
|
2024-01-22 13:09:35 +00:00
|
|
|
# add perf arguments
|
|
|
|
option(LLAMA_PERF "llama: enable perf" OFF)
|
|
|
|
|
2023-11-27 19:25:42 +00:00
|
|
|
# Required for relocatable CMake package
|
|
|
|
include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/build-info.cmake)
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
#
|
|
|
|
# Compile flags
|
|
|
|
#
|
2024-02-16 17:05:56 +00:00
|
|
|
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
if (LLAMA_SYCL)
|
|
|
|
set(CMAKE_CXX_STANDARD 17)
|
|
|
|
else()
|
|
|
|
set(CMAKE_CXX_STANDARD 11)
|
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
|
|
|
|
set(CMAKE_CXX_STANDARD_REQUIRED true)
|
2023-03-31 19:19:16 +00:00
|
|
|
set(CMAKE_C_STANDARD 11)
|
2023-03-21 00:37:16 +00:00
|
|
|
set(CMAKE_C_STANDARD_REQUIRED true)
|
|
|
|
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
find_package(Threads REQUIRED)
|
2023-09-13 13:08:52 +00:00
|
|
|
include(CheckCXXCompilerFlag)
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2024-02-17 21:03:14 +00:00
|
|
|
if (LLAMA_FATAL_WARNINGS)
|
|
|
|
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
|
|
|
add_compile_options(-Werror)
|
|
|
|
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
|
|
|
add_compile_options(/WX)
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
2023-12-01 18:18:35 +00:00
|
|
|
# enable libstdc++ assertions for debug builds
|
|
|
|
if (CMAKE_SYSTEM_NAME MATCHES "Linux")
|
|
|
|
add_compile_definitions($<$<CONFIG:Debug>:_GLIBCXX_ASSERTIONS>)
|
|
|
|
endif()
|
|
|
|
|
2023-03-13 17:12:33 +00:00
|
|
|
if (NOT MSVC)
|
|
|
|
if (LLAMA_SANITIZE_THREAD)
|
2023-03-21 00:37:16 +00:00
|
|
|
add_compile_options(-fsanitize=thread)
|
2024-02-16 17:05:56 +00:00
|
|
|
link_libraries (-fsanitize=thread)
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_SANITIZE_ADDRESS)
|
2023-03-21 00:37:16 +00:00
|
|
|
add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
|
2024-02-16 17:05:56 +00:00
|
|
|
link_libraries (-fsanitize=address)
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_SANITIZE_UNDEFINED)
|
2023-03-21 00:37:16 +00:00
|
|
|
add_compile_options(-fsanitize=undefined)
|
2024-02-16 17:05:56 +00:00
|
|
|
link_libraries (-fsanitize=undefined)
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
if (APPLE AND LLAMA_ACCELERATE)
|
2023-03-13 17:12:33 +00:00
|
|
|
find_library(ACCELERATE_FRAMEWORK Accelerate)
|
|
|
|
if (ACCELERATE_FRAMEWORK)
|
|
|
|
message(STATUS "Accelerate framework found")
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
add_compile_definitions(GGML_USE_ACCELERATE)
|
2023-09-27 15:34:32 +00:00
|
|
|
add_compile_definitions(ACCELERATE_NEW_LAPACK)
|
|
|
|
add_compile_definitions(ACCELERATE_LAPACK_ILP64)
|
2023-03-21 00:37:16 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${ACCELERATE_FRAMEWORK})
|
2023-03-13 17:12:33 +00:00
|
|
|
else()
|
|
|
|
message(WARNING "Accelerate framework not found")
|
|
|
|
endif()
|
|
|
|
endif()
|
2023-04-20 01:14:14 +00:00
|
|
|
|
2023-09-04 19:26:24 +00:00
|
|
|
if (LLAMA_METAL)
|
2024-01-02 08:57:44 +00:00
|
|
|
find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
|
|
|
|
find_library(METAL_FRAMEWORK Metal REQUIRED)
|
|
|
|
find_library(METALKIT_FRAMEWORK MetalKit REQUIRED)
|
2023-09-04 19:26:24 +00:00
|
|
|
|
|
|
|
message(STATUS "Metal framework found")
|
2023-09-14 17:04:40 +00:00
|
|
|
set(GGML_HEADERS_METAL ggml-metal.h)
|
|
|
|
set(GGML_SOURCES_METAL ggml-metal.m)
|
2023-09-04 19:26:24 +00:00
|
|
|
|
|
|
|
add_compile_definitions(GGML_USE_METAL)
|
2023-09-05 22:21:10 +00:00
|
|
|
if (LLAMA_METAL_NDEBUG)
|
|
|
|
add_compile_definitions(GGML_METAL_NDEBUG)
|
|
|
|
endif()
|
2023-09-04 19:26:24 +00:00
|
|
|
|
|
|
|
# get full path to the file
|
|
|
|
#add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
|
|
|
|
|
|
|
|
# copy ggml-metal.metal to bin directory
|
2023-11-30 21:44:11 +00:00
|
|
|
configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
|
2023-09-04 19:26:24 +00:00
|
|
|
|
2024-01-02 08:57:44 +00:00
|
|
|
if (LLAMA_METAL_SHADER_DEBUG)
|
|
|
|
# custom command to do the following:
|
|
|
|
# xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
|
2024-01-05 14:30:52 +00:00
|
|
|
# xcrun -sdk macosx metallib ggml-metal.air -o default.metallib
|
2024-01-02 08:57:44 +00:00
|
|
|
#
|
|
|
|
# note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works
|
|
|
|
# disabling fast math is needed in order to pass tests/test-backend-ops
|
|
|
|
# note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1
|
2024-01-05 14:30:52 +00:00
|
|
|
# note: unfortunately, we have to call it default.metallib instead of ggml.metallib
|
|
|
|
# ref: https://github.com/ggerganov/whisper.cpp/issues/1720
|
2024-01-02 08:57:44 +00:00
|
|
|
set(XC_FLAGS -fno-fast-math -fno-inline -g)
|
|
|
|
if (LLAMA_QKK_64)
|
|
|
|
set(XC_FLAGS ${XC_FLAGS} -DQK_K=64)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
add_custom_command(
|
2024-01-05 14:30:52 +00:00
|
|
|
OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
|
2024-01-02 08:57:44 +00:00
|
|
|
COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
|
2024-01-05 14:30:52 +00:00
|
|
|
COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
|
2024-01-02 08:57:44 +00:00
|
|
|
DEPENDS ggml-metal.metal
|
|
|
|
COMMENT "Compiling Metal kernels"
|
|
|
|
)
|
|
|
|
|
|
|
|
add_custom_target(
|
|
|
|
ggml-metal ALL
|
2024-01-05 14:30:52 +00:00
|
|
|
DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
|
2024-01-02 08:57:44 +00:00
|
|
|
)
|
|
|
|
endif()
|
|
|
|
|
2023-09-04 19:26:24 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
|
|
|
|
${FOUNDATION_LIBRARY}
|
|
|
|
${METAL_FRAMEWORK}
|
|
|
|
${METALKIT_FRAMEWORK}
|
|
|
|
)
|
|
|
|
endif()
|
2023-05-20 14:58:31 +00:00
|
|
|
if (LLAMA_BLAS)
|
2023-03-21 00:37:16 +00:00
|
|
|
if (LLAMA_STATIC)
|
|
|
|
set(BLA_STATIC ON)
|
|
|
|
endif()
|
2023-05-20 14:58:31 +00:00
|
|
|
if ($(CMAKE_VERSION) VERSION_GREATER_EQUAL 3.22)
|
|
|
|
set(BLA_SIZEOF_INTEGER 8)
|
|
|
|
endif()
|
2023-06-16 18:53:04 +00:00
|
|
|
|
2023-05-20 14:58:31 +00:00
|
|
|
set(BLA_VENDOR ${LLAMA_BLAS_VENDOR})
|
2023-03-21 00:37:16 +00:00
|
|
|
find_package(BLAS)
|
2023-06-16 18:53:04 +00:00
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
if (BLAS_FOUND)
|
2023-05-20 14:58:31 +00:00
|
|
|
message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
|
2023-03-21 00:37:16 +00:00
|
|
|
|
2023-06-16 18:53:04 +00:00
|
|
|
if ("${BLAS_INCLUDE_DIRS}" STREQUAL "")
|
|
|
|
# BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
|
|
|
|
# see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
|
|
|
|
find_package(PkgConfig REQUIRED)
|
|
|
|
if (${LLAMA_BLAS_VENDOR} MATCHES "Generic")
|
|
|
|
pkg_check_modules(DepBLAS REQUIRED blas)
|
|
|
|
elseif (${LLAMA_BLAS_VENDOR} MATCHES "OpenBLAS")
|
2024-01-05 16:04:40 +00:00
|
|
|
# As of openblas v0.3.22, the 64-bit is named openblas64.pc
|
|
|
|
pkg_check_modules(DepBLAS openblas64)
|
|
|
|
if (NOT DepBLAS_FOUND)
|
|
|
|
pkg_check_modules(DepBLAS REQUIRED openblas)
|
|
|
|
endif()
|
2023-06-16 18:53:04 +00:00
|
|
|
elseif (${LLAMA_BLAS_VENDOR} MATCHES "FLAME")
|
|
|
|
pkg_check_modules(DepBLAS REQUIRED blis)
|
|
|
|
elseif (${LLAMA_BLAS_VENDOR} MATCHES "ATLAS")
|
|
|
|
pkg_check_modules(DepBLAS REQUIRED blas-atlas)
|
|
|
|
elseif (${LLAMA_BLAS_VENDOR} MATCHES "FlexiBLAS")
|
|
|
|
pkg_check_modules(DepBLAS REQUIRED flexiblas_api)
|
|
|
|
elseif (${LLAMA_BLAS_VENDOR} MATCHES "Intel")
|
|
|
|
# all Intel* libraries share the same include path
|
2023-07-21 10:26:34 +00:00
|
|
|
pkg_check_modules(DepBLAS REQUIRED mkl-sdl)
|
2023-06-16 18:53:04 +00:00
|
|
|
elseif (${LLAMA_BLAS_VENDOR} MATCHES "NVHPC")
|
|
|
|
# this doesn't provide pkg-config
|
|
|
|
# suggest to assign BLAS_INCLUDE_DIRS on your own
|
|
|
|
if ("${NVHPC_VERSION}" STREQUAL "")
|
|
|
|
message(WARNING "Better to set NVHPC_VERSION")
|
|
|
|
else()
|
|
|
|
set(DepBLAS_FOUND ON)
|
|
|
|
set(DepBLAS_INCLUDE_DIRS "/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
if (DepBLAS_FOUND)
|
|
|
|
set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})
|
|
|
|
else()
|
|
|
|
message(WARNING "BLAS_INCLUDE_DIRS neither been provided nor been automatically"
|
|
|
|
" detected by pkgconfig, trying to find cblas.h from possible paths...")
|
|
|
|
find_path(BLAS_INCLUDE_DIRS
|
|
|
|
NAMES cblas.h
|
|
|
|
HINTS
|
|
|
|
/usr/include
|
|
|
|
/usr/local/include
|
|
|
|
/usr/include/openblas
|
|
|
|
/opt/homebrew/opt/openblas/include
|
|
|
|
/usr/local/opt/openblas/include
|
|
|
|
/usr/include/x86_64-linux-gnu/openblas/include
|
|
|
|
)
|
|
|
|
endif()
|
|
|
|
endif()
|
2023-06-15 17:51:26 +00:00
|
|
|
|
|
|
|
message(STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}")
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-05-20 14:58:31 +00:00
|
|
|
add_compile_options(${BLAS_LINKER_FLAGS})
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
add_compile_definitions(GGML_USE_OPENBLAS)
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-07-09 08:12:20 +00:00
|
|
|
if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${LLAMA_BLAS_VENDOR} MATCHES "Generic" OR ${LLAMA_BLAS_VENDOR} MATCHES "Intel"))
|
|
|
|
add_compile_definitions(GGML_BLAS_USE_MKL)
|
|
|
|
endif()
|
2023-05-20 14:58:31 +00:00
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${BLAS_LIBRARIES})
|
|
|
|
set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${BLAS_INCLUDE_DIRS})
|
2023-03-21 00:37:16 +00:00
|
|
|
else()
|
2023-05-20 14:58:31 +00:00
|
|
|
message(WARNING "BLAS not found, please refer to "
|
|
|
|
"https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
|
|
|
|
" to set correct LLAMA_BLAS_VENDOR")
|
2023-03-21 00:37:16 +00:00
|
|
|
endif()
|
|
|
|
endif()
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2023-10-29 16:32:28 +00:00
|
|
|
if (LLAMA_QKK_64)
|
|
|
|
add_compile_definitions(GGML_QKK_64)
|
k-quants : support for super-block size of 64 (#2001)
* k_quants: WIP super-blocks with 64 weights
* k_quants: WIP super-blocks with 64 weights
Q6_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q4_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q2_K scalar and AVX2 works. Q2_K is way too slow (it is actually slower
than the scalar implementation)
* k_quants: WIP super-blocks with 64 weights
Q3_K scalar and AVX2 works.
* k_quants: WIP super-blocks with 64 weights
Q5_K scalar and AVX2 works, and with that all
k_quants are done on AVX2 and scalar
* k_quants: WIP super-blocks with 64 weights
Q6_K working on CUDA. Cannot make it run quite as gast as
with super-blocks with 256 weigths: 8% slower on 4080,
20% slower on the 1660 (but there we fit 1 less layer on the
GPU because pf the larger model size), so some fraction of
these 20% is due to that,
* k_quants: WIP super-blocks with 64 weights
Q4_K working on CUDA. ~10% slower on GTX-1660,
16% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q2_K working on CUDA. ~3% slower on GTX-1660,
10% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q3_K working on CUDA.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on CUDA, and with this CUDA is done.
* k_quants: WIP super-blocks with 64 weights
Q6_K working on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Q4_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q2_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q3_K working on ARM_NEON, but quite a bit slower than 256 weights.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on ARM_NEON, but quite a bit slower than 256 weights.
With that, we have full support for ARM_NEON, although
performance is not quite there.
* k_quants: WIP super-blocks with 64 weights
Slightly more efficient Q3_K and Q5_K
* k_quants: WIP super-blocks with 64 weights
Another small improvement for Q3_K and Q5_K on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Yet another speedup for Q5_K on ARM_NEON.
We are now within 10% of the QK_K = 256 version.
* k_quants: WIP super-blocks with 64 weights
* We are able to pass preprocessor macros to the Metal
compiler
* Q6_K works and is actually slightly more efficient than
the QK_K = 256 version (25.2 ms vs 25.8 ms)
* k_quants: WIP super-blocks with 64 weights
Q4_K works on Metal and is actually slightly faster
than QK_K = 256 (21.95 ms vs 24.0 ms).
* k_quants: WIP super-blocks with 64 weights
Q2_K works on Metal and is very slightly faster
than QK_K = 256 (23.8 ms vs 24.2 ms).
* k_quants: WIP super-blocks with 64 weights
Q3_K works on Metal and is slightly faster
than QK_K = 256 (26.6 ms vs 28.3 ms).
* k_quants: WIP super-blocks with 64 weights
Q5_K works on Metal and is slightly faster
than QK_K = 256 (23.7 ms vs 26.3 ms).
* k_quants: call them _K, not _k, also on Metal
* k_quants: correctly define QK_K in llama.cpp
* Fixed bug in q4_K quantization added with the 64-block addition
* Simplify via lambda
* k_quants: swicth Q3_K to 4-bit scales when QK_K = 64
Otherwise there isn't much benefit from this
quantization type. There is some very slight loss
in accuracy, but we reduce size by ~7%.
E.g., for OpenLLaMA-3B, Q3_K_S perplexity is
8.6131 with 8-bit scales and 8.6352 with 4-bit,
while file size decreases from 1.53G to 1.44G.
* k_quants: switch Q4_K to 4-bit scales when QK_K = 64
Here the loss in accuracy is greater than for Q3_K,
but the Q4_K points still move further to the left on
the perplexity vs size curve.
* k_quants: forgot to add the Metal changes in last commit
* k_quants: change Q5_K to be type 0 when QK_K = 64
Still needs AVX2 implementation
* k_quants: AVX2 implementation for new 64-weight Q5_K
* k_quants: 10% faster ARM_NEON Q5_K dot product
* k_quants: fixed issue caused by merging with master
---------
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
2023-06-26 16:43:07 +00:00
|
|
|
endif()
|
|
|
|
|
2023-04-19 09:22:45 +00:00
|
|
|
if (LLAMA_CUBLAS)
|
|
|
|
cmake_minimum_required(VERSION 3.17)
|
|
|
|
|
|
|
|
find_package(CUDAToolkit)
|
|
|
|
if (CUDAToolkit_FOUND)
|
|
|
|
message(STATUS "cuBLAS found")
|
|
|
|
|
2023-04-20 01:14:14 +00:00
|
|
|
enable_language(CUDA)
|
|
|
|
|
2023-09-14 17:04:40 +00:00
|
|
|
set(GGML_HEADERS_CUDA ggml-cuda.h)
|
|
|
|
set(GGML_SOURCES_CUDA ggml-cuda.cu)
|
2023-04-20 01:14:14 +00:00
|
|
|
|
2023-04-19 09:22:45 +00:00
|
|
|
add_compile_definitions(GGML_USE_CUBLAS)
|
2023-07-05 12:19:42 +00:00
|
|
|
if (LLAMA_CUDA_FORCE_DMMV)
|
|
|
|
add_compile_definitions(GGML_CUDA_FORCE_DMMV)
|
|
|
|
endif()
|
2023-10-27 14:01:23 +00:00
|
|
|
if (LLAMA_CUDA_FORCE_MMQ)
|
|
|
|
add_compile_definitions(GGML_CUDA_FORCE_MMQ)
|
|
|
|
endif()
|
2023-05-25 21:07:29 +00:00
|
|
|
add_compile_definitions(GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
|
2023-07-05 12:19:42 +00:00
|
|
|
add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
|
|
|
|
if (DEFINED LLAMA_CUDA_DMMV_Y)
|
|
|
|
add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_DMMV_Y}) # for backwards compatibility
|
|
|
|
endif()
|
2023-07-31 17:52:22 +00:00
|
|
|
if (LLAMA_CUDA_F16 OR LLAMA_CUDA_DMMV_F16)
|
|
|
|
add_compile_definitions(GGML_CUDA_F16)
|
2023-06-19 08:23:56 +00:00
|
|
|
endif()
|
2023-06-16 17:08:44 +00:00
|
|
|
add_compile_definitions(K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER})
|
2023-09-17 14:37:53 +00:00
|
|
|
add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${LLAMA_CUDA_PEER_MAX_BATCH_SIZE})
|
2023-04-19 09:22:45 +00:00
|
|
|
|
|
|
|
if (LLAMA_STATIC)
|
2023-12-17 10:57:33 +00:00
|
|
|
if (WIN32)
|
|
|
|
# As of 12.3.1 CUDA Tookit for Windows does not offer a static cublas library
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas CUDA::cublasLt)
|
|
|
|
else ()
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
|
|
|
|
endif()
|
2023-04-19 09:22:45 +00:00
|
|
|
else()
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart CUDA::cublas CUDA::cublasLt)
|
|
|
|
endif()
|
|
|
|
|
2023-12-24 13:34:22 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver)
|
|
|
|
|
2023-06-21 21:49:25 +00:00
|
|
|
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
|
2023-07-31 13:44:35 +00:00
|
|
|
# 52 == lowest CUDA 12 standard
|
|
|
|
# 60 == f16 CUDA intrinsics
|
|
|
|
# 61 == integer CUDA intrinsics
|
2023-08-02 14:48:10 +00:00
|
|
|
# 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
|
|
|
|
if (LLAMA_CUDA_F16 OR LLAMA_CUDA_DMMV_F16)
|
2023-07-31 13:44:35 +00:00
|
|
|
set(CMAKE_CUDA_ARCHITECTURES "60;61;70") # needed for f16 CUDA intrinsics
|
2023-06-21 21:49:25 +00:00
|
|
|
else()
|
2023-07-31 13:44:35 +00:00
|
|
|
set(CMAKE_CUDA_ARCHITECTURES "52;61;70") # lowest CUDA 12 standard + lowest for integer intrinsics
|
2023-10-24 13:48:37 +00:00
|
|
|
#set(CMAKE_CUDA_ARCHITECTURES "") # use this to compile much faster, but only F16 models work
|
2023-06-21 21:49:25 +00:00
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
|
|
|
|
|
2023-04-19 09:22:45 +00:00
|
|
|
else()
|
|
|
|
message(WARNING "cuBLAS not found")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
2023-07-10 15:49:56 +00:00
|
|
|
if (LLAMA_MPI)
|
|
|
|
cmake_minimum_required(VERSION 3.10)
|
|
|
|
find_package(MPI)
|
|
|
|
if (MPI_C_FOUND)
|
|
|
|
message(STATUS "MPI found")
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-09-14 17:04:40 +00:00
|
|
|
set(GGML_HEADERS_MPI ggml-mpi.h)
|
2024-02-16 17:05:56 +00:00
|
|
|
set(GGML_SOURCES_MPI ggml-mpi.c)
|
|
|
|
|
2023-07-10 15:49:56 +00:00
|
|
|
add_compile_definitions(GGML_USE_MPI)
|
|
|
|
add_compile_definitions(${MPI_C_COMPILE_DEFINITIONS})
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-10-02 13:16:50 +00:00
|
|
|
if (NOT MSVC)
|
|
|
|
add_compile_options(-Wno-cast-qual)
|
|
|
|
endif()
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-07-10 15:49:56 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${MPI_C_LIBRARIES})
|
|
|
|
set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${MPI_C_INCLUDE_DIRS})
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-07-10 15:49:56 +00:00
|
|
|
# Even if you're only using the C header, C++ programs may bring in MPI
|
|
|
|
# C++ functions, so more linkage is needed
|
|
|
|
if (MPI_CXX_FOUND)
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${MPI_CXX_LIBRARIES})
|
|
|
|
endif()
|
|
|
|
else()
|
|
|
|
message(WARNING "MPI not found")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
ggml : add CLBlast support (#1164)
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing
* Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers
* Finish merge of ClBlast support
* Move CLBlast implementation to separate file
Add buffer reuse code (adapted from slaren's cuda implementation)
* Add q4_2 and q4_3 CLBlast support, improve code
* Double CLBlast speed by disabling OpenBLAS thread workaround
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
* Fix device selection env variable names
* Fix cast in opencl kernels
* Add CLBlast to CMakeLists.txt
* Replace buffer pool with static buffers a, b, qb, c
Fix compile warnings
* Fix typos, use GGML_TYPE defines, improve code
* Improve btype dequant kernel selection code, add error if type is unsupported
* Improve code quality
* Move internal stuff out of header
* Use internal enums instead of CLBlast enums
* Remove leftover C++ includes and defines
* Make event use easier to read
Co-authored-by: Henri Vasserman <henv@hot.ee>
* Use c compiler for opencl files
* Simplify code, fix include
* First check error, then release event
* Make globals static, fix indentation
* Rename dequant kernels file to conform with other file names
* Fix import cl file name
---------
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-04-28 14:57:16 +00:00
|
|
|
if (LLAMA_CLBLAST)
|
|
|
|
find_package(CLBlast)
|
|
|
|
if (CLBlast_FOUND)
|
|
|
|
message(STATUS "CLBlast found")
|
|
|
|
|
2023-09-14 17:04:40 +00:00
|
|
|
set(GGML_HEADERS_OPENCL ggml-opencl.h)
|
|
|
|
set(GGML_SOURCES_OPENCL ggml-opencl.cpp)
|
ggml : add CLBlast support (#1164)
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing
* Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers
* Finish merge of ClBlast support
* Move CLBlast implementation to separate file
Add buffer reuse code (adapted from slaren's cuda implementation)
* Add q4_2 and q4_3 CLBlast support, improve code
* Double CLBlast speed by disabling OpenBLAS thread workaround
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
* Fix device selection env variable names
* Fix cast in opencl kernels
* Add CLBlast to CMakeLists.txt
* Replace buffer pool with static buffers a, b, qb, c
Fix compile warnings
* Fix typos, use GGML_TYPE defines, improve code
* Improve btype dequant kernel selection code, add error if type is unsupported
* Improve code quality
* Move internal stuff out of header
* Use internal enums instead of CLBlast enums
* Remove leftover C++ includes and defines
* Make event use easier to read
Co-authored-by: Henri Vasserman <henv@hot.ee>
* Use c compiler for opencl files
* Simplify code, fix include
* First check error, then release event
* Make globals static, fix indentation
* Rename dequant kernels file to conform with other file names
* Fix import cl file name
---------
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-04-28 14:57:16 +00:00
|
|
|
|
|
|
|
add_compile_definitions(GGML_USE_CLBLAST)
|
|
|
|
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
|
|
|
|
else()
|
|
|
|
message(WARNING "CLBlast not found")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
ggml : add Vulkan backend (#2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
if (LLAMA_VULKAN)
|
|
|
|
find_package(Vulkan)
|
|
|
|
if (Vulkan_FOUND)
|
|
|
|
message(STATUS "Vulkan found")
|
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
set(GGML_HEADERS_VULKAN ggml-vulkan.h)
|
|
|
|
set(GGML_SOURCES_VULKAN ggml-vulkan.cpp)
|
ggml : add Vulkan backend (#2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
|
|
|
|
add_compile_definitions(GGML_USE_VULKAN)
|
|
|
|
|
2024-02-03 17:15:00 +00:00
|
|
|
if (LLAMA_VULKAN_CHECK_RESULTS)
|
2024-02-16 17:05:56 +00:00
|
|
|
add_compile_definitions(GGML_VULKAN_CHECK_RESULTS)
|
2024-02-03 17:15:00 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_VULKAN_DEBUG)
|
2024-02-16 17:05:56 +00:00
|
|
|
add_compile_definitions(GGML_VULKAN_DEBUG)
|
2024-02-03 17:15:00 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_VULKAN_VALIDATE)
|
2024-02-16 17:05:56 +00:00
|
|
|
add_compile_definitions(GGML_VULKAN_VALIDATE)
|
2024-02-03 17:15:00 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_VULKAN_RUN_TESTS)
|
2024-02-16 17:05:56 +00:00
|
|
|
add_compile_definitions(GGML_VULKAN_RUN_TESTS)
|
2024-02-03 17:15:00 +00:00
|
|
|
endif()
|
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} Vulkan::Vulkan)
|
ggml : add Vulkan backend (#2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
else()
|
|
|
|
message(WARNING "Vulkan not found")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
2023-08-25 09:09:42 +00:00
|
|
|
if (LLAMA_HIPBLAS)
|
|
|
|
list(APPEND CMAKE_PREFIX_PATH /opt/rocm)
|
|
|
|
|
|
|
|
if (NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang")
|
|
|
|
message(WARNING "Only LLVM is supported for HIP, hint: CC=/opt/rocm/llvm/bin/clang")
|
|
|
|
endif()
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-08-25 09:09:42 +00:00
|
|
|
if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
|
|
|
|
message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++")
|
|
|
|
endif()
|
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
find_package(hip REQUIRED)
|
|
|
|
find_package(hipblas REQUIRED)
|
|
|
|
find_package(rocblas REQUIRED)
|
2023-08-25 09:09:42 +00:00
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
message(STATUS "HIP and hipBLAS found")
|
2023-08-25 09:09:42 +00:00
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
set(GGML_HEADERS_ROCM ggml-cuda.h)
|
|
|
|
set(GGML_SOURCES_ROCM ggml-cuda.cu)
|
|
|
|
|
|
|
|
add_compile_definitions(GGML_USE_HIPBLAS GGML_USE_CUBLAS)
|
|
|
|
|
|
|
|
if (LLAMA_HIP_UMA)
|
|
|
|
add_compile_definitions(GGML_HIP_UMA)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_CUDA_FORCE_DMMV)
|
|
|
|
add_compile_definitions(GGML_CUDA_FORCE_DMMV)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_CUDA_FORCE_MMQ)
|
|
|
|
add_compile_definitions(GGML_CUDA_FORCE_MMQ)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
add_compile_definitions(GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
|
|
|
|
add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
|
|
|
|
add_compile_definitions(K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER})
|
|
|
|
|
|
|
|
set_source_files_properties(ggml-cuda.cu PROPERTIES LANGUAGE CXX)
|
|
|
|
|
|
|
|
if (LLAMA_STATIC)
|
|
|
|
message(FATAL_ERROR "Static linking not supported for HIP/ROCm")
|
2023-08-25 09:09:42 +00:00
|
|
|
endif()
|
2024-02-16 17:05:56 +00:00
|
|
|
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} hip::device PUBLIC hip::host roc::rocblas roc::hipblas)
|
2023-08-25 09:09:42 +00:00
|
|
|
endif()
|
|
|
|
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
if (LLAMA_SYCL)
|
|
|
|
if ( NOT DEFINED ENV{ONEAPI_ROOT})
|
|
|
|
message(FATAL_ERROR "Not detect ENV {ONEAPI_ROOT}, please install oneAPI & source it, like: source /opt/intel/oneapi/setvars.sh")
|
|
|
|
endif()
|
|
|
|
#todo: AOT
|
|
|
|
|
|
|
|
find_package(IntelSYCL REQUIRED)
|
2024-02-16 17:05:56 +00:00
|
|
|
|
|
|
|
message(STATUS "SYCL found")
|
|
|
|
|
|
|
|
add_compile_definitions(GML_USE_SYCL)
|
|
|
|
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
if (LLAMA_SYCL_F16)
|
|
|
|
add_compile_definitions(GGML_SYCL_F16)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
add_compile_options(-I./) #include DPCT
|
|
|
|
add_compile_options(-I/${SYCL_INCLUDE_DIR})
|
|
|
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing")
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl -L${MKLROOT}/lib")
|
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
set(GGML_HEADERS_SYCL ggml-sycl.h)
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
set(GGML_SOURCES_SYCL ggml-sycl.cpp)
|
|
|
|
|
2024-01-31 02:38:07 +00:00
|
|
|
if (WIN32)
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl sycl7 OpenCL mkl_sycl_blas_dll.lib mkl_intel_ilp64_dll.lib mkl_sequential_dll.lib mkl_core_dll.lib)
|
|
|
|
else()
|
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread)
|
|
|
|
endif()
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
endif()
|
|
|
|
|
2024-01-29 20:50:50 +00:00
|
|
|
if (LLAMA_KOMPUTE)
|
|
|
|
add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
|
|
|
|
find_package(Vulkan COMPONENTS glslc REQUIRED)
|
|
|
|
find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
|
|
|
|
if (NOT glslc_executable)
|
|
|
|
message(FATAL_ERROR "glslc not found")
|
|
|
|
endif()
|
|
|
|
|
|
|
|
function(compile_shader)
|
2024-02-16 17:05:56 +00:00
|
|
|
set(options)
|
|
|
|
set(oneValueArgs)
|
|
|
|
set(multiValueArgs SOURCES)
|
|
|
|
cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
|
|
|
foreach(source ${compile_shader_SOURCES})
|
|
|
|
get_filename_component(filename ${source} NAME)
|
|
|
|
set(spv_file ${filename}.spv)
|
2024-01-29 20:50:50 +00:00
|
|
|
add_custom_command(
|
2024-02-16 17:05:56 +00:00
|
|
|
OUTPUT ${spv_file}
|
|
|
|
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source}
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/common.comp
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_getrows.comp
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n.comp
|
|
|
|
COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR}/${source}
|
|
|
|
COMMENT "Compiling ${source} to ${spv_file}"
|
|
|
|
)
|
|
|
|
|
|
|
|
get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
|
|
|
|
set(FILE_NAME "shader${RAW_FILE_NAME}")
|
|
|
|
string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
|
|
|
|
string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
|
|
|
|
string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
|
|
|
|
set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
|
|
|
|
message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
|
|
|
|
if(CMAKE_GENERATOR MATCHES "Visual Studio")
|
|
|
|
add_custom_command(
|
|
|
|
OUTPUT ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
DEPENDS ${spv_file} xxd
|
|
|
|
COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
|
|
|
|
)
|
|
|
|
else()
|
|
|
|
add_custom_command(
|
|
|
|
OUTPUT ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
|
|
|
|
DEPENDS ${spv_file} xxd
|
|
|
|
COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
|
|
|
|
)
|
|
|
|
endif()
|
|
|
|
endforeach()
|
2024-01-29 20:50:50 +00:00
|
|
|
endfunction()
|
|
|
|
|
|
|
|
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/kompute/CMakeLists.txt")
|
|
|
|
message(STATUS "Kompute found")
|
|
|
|
set(KOMPUTE_OPT_LOG_LEVEL Error CACHE STRING "Kompute log level")
|
|
|
|
add_subdirectory(kompute)
|
|
|
|
|
|
|
|
# Compile our shaders
|
|
|
|
compile_shader(SOURCES
|
2024-02-16 17:05:56 +00:00
|
|
|
kompute-shaders/op_scale.comp
|
|
|
|
kompute-shaders/op_scale_8.comp
|
|
|
|
kompute-shaders/op_add.comp
|
|
|
|
kompute-shaders/op_addrow.comp
|
|
|
|
kompute-shaders/op_mul.comp
|
|
|
|
kompute-shaders/op_silu.comp
|
|
|
|
kompute-shaders/op_relu.comp
|
|
|
|
kompute-shaders/op_gelu.comp
|
|
|
|
kompute-shaders/op_softmax.comp
|
|
|
|
kompute-shaders/op_norm.comp
|
|
|
|
kompute-shaders/op_rmsnorm.comp
|
|
|
|
kompute-shaders/op_diagmask.comp
|
|
|
|
kompute-shaders/op_mul_mat_mat_f32.comp
|
|
|
|
kompute-shaders/op_mul_mat_f16.comp
|
|
|
|
kompute-shaders/op_mul_mat_q8_0.comp
|
|
|
|
kompute-shaders/op_mul_mat_q4_0.comp
|
|
|
|
kompute-shaders/op_mul_mat_q4_1.comp
|
|
|
|
kompute-shaders/op_mul_mat_q6_k.comp
|
|
|
|
kompute-shaders/op_getrows_f16.comp
|
|
|
|
kompute-shaders/op_getrows_q4_0.comp
|
|
|
|
kompute-shaders/op_getrows_q4_1.comp
|
|
|
|
kompute-shaders/op_getrows_q6_k.comp
|
|
|
|
kompute-shaders/op_rope_f16.comp
|
|
|
|
kompute-shaders/op_rope_f32.comp
|
|
|
|
kompute-shaders/op_cpy_f16_f16.comp
|
|
|
|
kompute-shaders/op_cpy_f16_f32.comp
|
|
|
|
kompute-shaders/op_cpy_f32_f16.comp
|
|
|
|
kompute-shaders/op_cpy_f32_f32.comp
|
2024-01-29 20:50:50 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
# Create a custom target for our generated shaders
|
|
|
|
add_custom_target(generated_shaders DEPENDS
|
2024-02-16 17:05:56 +00:00
|
|
|
shaderop_scale.h
|
|
|
|
shaderop_scale_8.h
|
|
|
|
shaderop_add.h
|
|
|
|
shaderop_addrow.h
|
|
|
|
shaderop_mul.h
|
|
|
|
shaderop_silu.h
|
|
|
|
shaderop_relu.h
|
|
|
|
shaderop_gelu.h
|
|
|
|
shaderop_softmax.h
|
|
|
|
shaderop_norm.h
|
|
|
|
shaderop_rmsnorm.h
|
|
|
|
shaderop_diagmask.h
|
|
|
|
shaderop_mul_mat_mat_f32.h
|
|
|
|
shaderop_mul_mat_f16.h
|
|
|
|
shaderop_mul_mat_q8_0.h
|
|
|
|
shaderop_mul_mat_q4_0.h
|
|
|
|
shaderop_mul_mat_q4_1.h
|
|
|
|
shaderop_mul_mat_q6_k.h
|
|
|
|
shaderop_getrows_f16.h
|
|
|
|
shaderop_getrows_q4_0.h
|
|
|
|
shaderop_getrows_q4_1.h
|
|
|
|
shaderop_getrows_q6_k.h
|
|
|
|
shaderop_rope_f16.h
|
|
|
|
shaderop_rope_f32.h
|
|
|
|
shaderop_cpy_f16_f16.h
|
|
|
|
shaderop_cpy_f16_f32.h
|
|
|
|
shaderop_cpy_f32_f16.h
|
|
|
|
shaderop_cpy_f32_f32.h
|
2024-01-29 20:50:50 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
# Create a custom command that depends on the generated_shaders
|
|
|
|
add_custom_command(
|
|
|
|
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
|
|
|
|
COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
|
|
|
|
DEPENDS generated_shaders
|
|
|
|
COMMENT "Ensuring shaders are generated before compiling ggml-kompute.cpp"
|
|
|
|
)
|
|
|
|
|
|
|
|
# Add the stamp to the main sources to ensure dependency tracking
|
|
|
|
set(GGML_SOURCES_KOMPUTE ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
|
2024-02-16 17:05:56 +00:00
|
|
|
set(GGML_HEADERS_KOMPUTE ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
|
|
|
|
|
2024-01-29 20:50:50 +00:00
|
|
|
add_compile_definitions(GGML_USE_KOMPUTE)
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2024-01-29 20:50:50 +00:00
|
|
|
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
|
|
|
|
set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
|
|
|
|
else()
|
|
|
|
message(WARNING "Kompute not found")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
2024-02-16 17:05:56 +00:00
|
|
|
if (LLAMA_CPU_HBM)
|
|
|
|
find_library(memkind memkind REQUIRED)
|
|
|
|
|
|
|
|
add_compile_definitions(GGML_USE_CPU_HBM)
|
|
|
|
|
|
|
|
target_link_libraries(ggml PUBLIC memkind)
|
|
|
|
endif()
|
|
|
|
|
|
|
|
if (LLAMA_PERF)
|
|
|
|
add_compile_definitions(GGML_PERF)
|
|
|
|
endif()
|
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
function(get_flags CCID CCVER)
|
|
|
|
set(C_FLAGS "")
|
|
|
|
set(CXX_FLAGS "")
|
|
|
|
|
|
|
|
if (CCID MATCHES "Clang")
|
|
|
|
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
|
|
|
|
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
|
|
|
|
|
|
|
|
if (
|
|
|
|
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
|
|
|
|
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
|
|
|
|
)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND C_FLAGS -Wdouble-promotion)
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
|
|
|
elseif (CCID STREQUAL "GNU")
|
|
|
|
set(C_FLAGS -Wdouble-promotion)
|
|
|
|
set(CXX_FLAGS -Wno-array-bounds)
|
|
|
|
|
|
|
|
if (CCVER VERSION_GREATER_EQUAL 7.1.0)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND CXX_FLAGS -Wno-format-truncation)
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
|
|
|
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND CXX_FLAGS -Wextra-semi)
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
2024-01-23 07:11:39 +00:00
|
|
|
elseif (CCID MATCHES "Intel")
|
ggml : add unified SYCL backend for Intel GPUs (#2690)
* first update for migration
* update init_cublas
* add debug functio, commit all help code
* step 1
* step 2
* step3 add fp16, slower 31->28
* add GGML_LIST_DEVICE function
* step 5 format device and print
* step6, enhance error check, remove CUDA macro, enhance device id to fix none-zero id issue
* support main device is non-zero
* step7 add debug for code path, rm log
* step 8, rename all macro & func from cuda by sycl
* fix error of select non-zero device, format device list
* ren ggml-sycl.hpp -> ggml-sycl.h
* clear CMAKE to rm unused lib and options
* correct queue: rm dtct:get_queue
* add print tensor function to debug
* fix error: wrong result in 658746bb26702e50f2c59c0e4ada8e9da6010481
* summary dpct definition in one header file to replace folder:dpct
* refactor device log
* mv dpct definition from folder dpct to ggml-sycl.h
* update readme, refactor build script
* fix build with sycl
* set nthread=1 when sycl, increase performance
* add run script, comment debug code
* add ls-sycl-device tool
* add ls-sycl-device, rm unused files
* rm rear space
* dos2unix
* Update README_sycl.md
* fix return type
* remove sycl version from include path
* restore rm code to fix hang issue
* add syc and link for sycl readme
* rm original sycl code before refactor
* fix code err
* add know issue for pvc hang issue
* enable SYCL_F16 support
* align pr4766
* check for sycl blas, better performance
* cleanup 1
* remove extra endif
* add build&run script, clean CMakefile, update guide by review comments
* rename macro to intel hardware
* editor config format
* format fixes
* format fixes
* editor format fix
* Remove unused headers
* skip build sycl tool for other code path
* replace tab by space
* fix blas matmul function
* fix mac build
* restore hip dependency
* fix conflict
* ren as review comments
* mv internal function to .cpp file
* export funciton print_sycl_devices(), mv class dpct definition to source file
* update CI/action for sycl code, fix CI error of repeat/dup
* fix action ID format issue
* rm unused strategy
* enable llama_f16 in ci
* fix conflict
* fix build break on MacOS, due to CI of MacOS depend on external ggml, instead of internal ggml
* fix ci cases for unsupported data type
* revert unrelated changed in cuda cmake
remove useless nommq
fix typo of GGML_USE_CLBLAS_SYCL
* revert hip cmake changes
* fix indent
* add prefix in func name
* revert no mmq
* rm cpu blas duplicate
* fix no_new_line
* fix src1->type==F16 bug.
* pass batch offset for F16 src1
* fix batch error
* fix wrong code
* revert sycl checking in test-sampling
* pass void as arguments of ggml_backend_sycl_print_sycl_devices
* remove extra blank line in test-sampling
* revert setting n_threads in sycl
* implement std::isinf for icpx with fast math.
* Update ci/run.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update examples/sycl/run-llama2.sh
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update CMakeLists.txt
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* add copyright and MIT license declare
* update the cmd example
---------
Co-authored-by: jianyuzh <jianyu.zhang@intel.com>
Co-authored-by: luoyu-intel <yu.luo@intel.com>
Co-authored-by: Meng, Hengyu <hengyu.meng@intel.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 15:56:23 +00:00
|
|
|
if (NOT LLAMA_SYCL)
|
|
|
|
# enable max optimization level when using Intel compiler
|
|
|
|
set(C_FLAGS -ipo -O3 -static -fp-model=fast -flto -fno-stack-protector)
|
|
|
|
set(CXX_FLAGS -ipo -O3 -static -fp-model=fast -flto -fno-stack-protector)
|
|
|
|
add_link_options(-fuse-ld=lld -static-intel)
|
|
|
|
endif()
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
|
|
|
|
|
|
|
set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)
|
|
|
|
set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
|
|
|
|
endfunction()
|
|
|
|
|
2023-03-13 17:12:33 +00:00
|
|
|
if (LLAMA_ALL_WARNINGS)
|
|
|
|
if (NOT MSVC)
|
2023-12-13 17:10:10 +00:00
|
|
|
set(WARNING_FLAGS -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function)
|
|
|
|
set(C_FLAGS -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes
|
|
|
|
-Werror=implicit-int -Werror=implicit-function-declaration)
|
|
|
|
set(CXX_FLAGS -Wmissing-declarations -Wmissing-noreturn)
|
2023-09-28 21:41:44 +00:00
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
set(C_FLAGS ${WARNING_FLAGS} ${C_FLAGS})
|
|
|
|
set(CXX_FLAGS ${WARNING_FLAGS} ${CXX_FLAGS})
|
|
|
|
|
|
|
|
get_flags(${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})
|
|
|
|
|
|
|
|
add_compile_options("$<$<COMPILE_LANGUAGE:C>:${C_FLAGS};${GF_C_FLAGS}>"
|
|
|
|
"$<$<COMPILE_LANGUAGE:CXX>:${CXX_FLAGS};${GF_CXX_FLAGS}>")
|
2023-03-13 17:12:33 +00:00
|
|
|
else()
|
|
|
|
# todo : msvc
|
2023-12-13 17:10:10 +00:00
|
|
|
set(C_FLAGS "")
|
|
|
|
set(CXX_FLAGS "")
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
|
2024-01-26 20:34:06 +00:00
|
|
|
set(CUDA_CXX_FLAGS "")
|
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
if (LLAMA_CUBLAS)
|
|
|
|
set(CUDA_FLAGS ${CXX_FLAGS} -use_fast_math)
|
|
|
|
if (NOT MSVC)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND CUDA_FLAGS -Wno-pedantic)
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
2023-10-02 13:16:50 +00:00
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
if (LLAMA_ALL_WARNINGS AND NOT MSVC)
|
|
|
|
set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c)
|
|
|
|
if (NOT CMAKE_CUDA_HOST_COMPILER STREQUAL "")
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND NVCC_CMD -ccbin ${CMAKE_CUDA_HOST_COMPILER})
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
execute_process(
|
|
|
|
COMMAND ${NVCC_CMD} -Xcompiler --version
|
|
|
|
OUTPUT_VARIABLE CUDA_CCFULLVER
|
|
|
|
ERROR_QUIET
|
|
|
|
)
|
2023-10-02 13:16:50 +00:00
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
if (NOT CUDA_CCFULLVER MATCHES clang)
|
|
|
|
set(CUDA_CCID "GNU")
|
|
|
|
execute_process(
|
|
|
|
COMMAND ${NVCC_CMD} -Xcompiler "-dumpfullversion -dumpversion"
|
|
|
|
OUTPUT_VARIABLE CUDA_CCVER
|
|
|
|
ERROR_QUIET
|
|
|
|
)
|
|
|
|
else()
|
|
|
|
if (CUDA_CCFULLVER MATCHES Apple)
|
|
|
|
set(CUDA_CCID "AppleClang")
|
|
|
|
else()
|
|
|
|
set(CUDA_CCID "Clang")
|
|
|
|
endif()
|
|
|
|
string(REGEX REPLACE "^.* version ([0-9.]*).*$" "\\1" CUDA_CCVER ${CUDA_CCFULLVER})
|
|
|
|
endif()
|
|
|
|
|
|
|
|
message("-- CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}")
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2023-12-13 17:10:10 +00:00
|
|
|
get_flags(${CUDA_CCID} ${CUDA_CCVER})
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND CUDA_CXX_FLAGS ${GF_CXX_FLAGS}) # This is passed to -Xcompiler later
|
2023-12-13 17:10:10 +00:00
|
|
|
endif()
|
|
|
|
endif()
|
2023-10-02 13:16:50 +00:00
|
|
|
|
2023-09-15 12:24:30 +00:00
|
|
|
if (WIN32)
|
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
|
|
|
add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
|
2023-04-22 08:18:20 +00:00
|
|
|
|
|
|
|
if (BUILD_SHARED_LIBS)
|
|
|
|
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
|
|
|
|
endif()
|
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
|
|
|
endif()
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
if (LLAMA_LTO)
|
|
|
|
include(CheckIPOSupported)
|
|
|
|
check_ipo_supported(RESULT result OUTPUT output)
|
|
|
|
if (result)
|
|
|
|
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
|
|
|
|
else()
|
|
|
|
message(WARNING "IPO is not supported: ${output}")
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
|
2024-01-20 08:11:31 +00:00
|
|
|
if (LLAMA_CCACHE)
|
|
|
|
find_program(LLAMA_CCACHE_FOUND ccache)
|
|
|
|
if (LLAMA_CCACHE_FOUND)
|
|
|
|
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
|
|
|
|
set(ENV{CCACHE_SLOPPINESS} time_macros)
|
2024-02-05 18:33:00 +00:00
|
|
|
message(STATUS "ccache found, compilation results will be cached. Disable with LLAMA_CCACHE=OFF.")
|
2024-01-20 08:11:31 +00:00
|
|
|
else()
|
2024-02-05 18:33:00 +00:00
|
|
|
message(STATUS "Warning: ccache not found - consider installing it for faster compilation or disable this warning with LLAMA_CCACHE=OFF")
|
2024-01-20 08:11:31 +00:00
|
|
|
endif ()
|
|
|
|
endif()
|
|
|
|
|
2023-11-14 17:34:41 +00:00
|
|
|
# this version of Apple ld64 is buggy
|
|
|
|
execute_process(
|
|
|
|
COMMAND ${CMAKE_C_COMPILER} ${CMAKE_EXE_LINKER_FLAGS} -Wl,-v
|
|
|
|
ERROR_VARIABLE output
|
2023-12-13 17:10:10 +00:00
|
|
|
OUTPUT_QUIET
|
2023-11-14 17:34:41 +00:00
|
|
|
)
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-11-14 17:34:41 +00:00
|
|
|
if (output MATCHES "dyld-1015\.7")
|
|
|
|
add_compile_definitions(HAVE_BUGGY_APPLE_LINKER)
|
|
|
|
endif()
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
# Architecture specific
|
|
|
|
# TODO: probably these flags need to be tweaked on some architectures
|
|
|
|
# feel free to update the Makefile for your architecture and send a pull request or issue
|
2023-03-13 17:12:33 +00:00
|
|
|
message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
|
2023-09-13 01:54:20 +00:00
|
|
|
if (MSVC)
|
2024-02-16 17:05:56 +00:00
|
|
|
string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR)
|
|
|
|
message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}")
|
2023-09-13 01:54:20 +00:00
|
|
|
else ()
|
2024-02-16 17:05:56 +00:00
|
|
|
set(CMAKE_GENERATOR_PLATFORM_LWR "")
|
2023-09-13 01:54:20 +00:00
|
|
|
endif ()
|
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
if (NOT MSVC)
|
|
|
|
if (LLAMA_STATIC)
|
|
|
|
add_link_options(-static)
|
|
|
|
if (MINGW)
|
|
|
|
add_link_options(-static-libgcc -static-libstdc++)
|
|
|
|
endif()
|
|
|
|
endif()
|
|
|
|
if (LLAMA_GPROF)
|
|
|
|
add_compile_options(-pg)
|
|
|
|
endif()
|
|
|
|
endif()
|
2023-03-13 17:12:33 +00:00
|
|
|
|
2024-01-26 20:34:06 +00:00
|
|
|
set(ARCH_FLAGS "")
|
2024-01-15 18:40:48 +00:00
|
|
|
|
2024-02-07 21:39:23 +00:00
|
|
|
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
|
|
|
|
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
|
|
|
|
CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
|
2023-03-13 17:12:33 +00:00
|
|
|
message(STATUS "ARM detected")
|
2023-03-21 00:37:16 +00:00
|
|
|
if (MSVC)
|
2024-02-14 08:49:01 +00:00
|
|
|
add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
|
2023-09-13 01:54:20 +00:00
|
|
|
add_compile_definitions(__ARM_NEON)
|
|
|
|
add_compile_definitions(__ARM_FEATURE_FMA)
|
2024-02-14 08:49:01 +00:00
|
|
|
|
|
|
|
set(CMAKE_REQUIRED_FLAGS_PREV ${CMAKE_REQUIRED_FLAGS})
|
|
|
|
string(JOIN " " CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "/arch:armv8.2")
|
|
|
|
check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD)
|
|
|
|
if (GGML_COMPILER_SUPPORT_DOTPROD)
|
|
|
|
add_compile_definitions(__ARM_FEATURE_DOTPROD)
|
|
|
|
endif ()
|
|
|
|
check_cxx_source_compiles("#include <arm_neon.h>\nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
|
|
|
|
if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
|
|
|
|
add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
|
|
|
|
endif ()
|
|
|
|
set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_PREV})
|
2023-03-21 00:37:16 +00:00
|
|
|
else()
|
2023-09-13 13:08:52 +00:00
|
|
|
check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
|
|
|
|
if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mfp16-format=ieee)
|
2023-09-13 13:08:52 +00:00
|
|
|
endif()
|
2023-04-30 18:48:38 +00:00
|
|
|
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
|
|
|
|
# Raspberry Pi 1, Zero
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access)
|
2023-04-30 18:48:38 +00:00
|
|
|
endif()
|
|
|
|
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
|
|
|
|
# Raspberry Pi 2
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
|
2023-04-30 18:48:38 +00:00
|
|
|
endif()
|
|
|
|
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
|
|
|
|
# Raspberry Pi 3, 4, Zero 2 (32-bit)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mno-unaligned-access)
|
2023-04-30 18:48:38 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
endif()
|
2024-02-07 21:39:23 +00:00
|
|
|
elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
|
|
|
|
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
|
|
|
|
CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$"))
|
2023-03-13 17:12:33 +00:00
|
|
|
message(STATUS "x86 detected")
|
|
|
|
if (MSVC)
|
2023-11-05 08:03:09 +00:00
|
|
|
# instruction set detection for MSVC only
|
|
|
|
if (LLAMA_NATIVE)
|
|
|
|
include(cmake/FindSIMD.cmake)
|
|
|
|
endif ()
|
2023-03-25 21:38:11 +00:00
|
|
|
if (LLAMA_AVX512)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS /arch:AVX512)
|
2023-04-17 13:10:57 +00:00
|
|
|
# MSVC has no compile-time flags enabling specific
|
|
|
|
# AVX512 extensions, neither it defines the
|
|
|
|
# macros corresponding to the extensions.
|
|
|
|
# Do it manually.
|
|
|
|
if (LLAMA_AVX512_VBMI)
|
2023-04-20 01:14:14 +00:00
|
|
|
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
|
|
|
|
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
|
2023-04-17 13:10:57 +00:00
|
|
|
endif()
|
|
|
|
if (LLAMA_AVX512_VNNI)
|
2023-04-20 01:14:14 +00:00
|
|
|
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
|
|
|
|
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
|
2023-04-17 13:10:57 +00:00
|
|
|
endif()
|
2023-03-25 21:38:11 +00:00
|
|
|
elseif (LLAMA_AVX2)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS /arch:AVX2)
|
2023-03-21 00:37:16 +00:00
|
|
|
elseif (LLAMA_AVX)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS /arch:AVX)
|
2023-03-21 00:37:16 +00:00
|
|
|
endif()
|
2023-03-13 17:12:33 +00:00
|
|
|
else()
|
2023-10-03 16:53:15 +00:00
|
|
|
if (LLAMA_NATIVE)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -march=native)
|
2023-10-03 16:53:15 +00:00
|
|
|
endif()
|
2023-04-13 12:48:21 +00:00
|
|
|
if (LLAMA_F16C)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mf16c)
|
2023-04-13 12:48:21 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
if (LLAMA_FMA)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mfma)
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
if (LLAMA_AVX)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mavx)
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
if (LLAMA_AVX2)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mavx2)
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
2023-03-25 21:38:11 +00:00
|
|
|
if (LLAMA_AVX512)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mavx512f)
|
|
|
|
list(APPEND ARCH_FLAGS -mavx512bw)
|
2023-04-17 13:10:57 +00:00
|
|
|
endif()
|
|
|
|
if (LLAMA_AVX512_VBMI)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mavx512vbmi)
|
2023-04-17 13:10:57 +00:00
|
|
|
endif()
|
|
|
|
if (LLAMA_AVX512_VNNI)
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mavx512vnni)
|
2023-03-25 21:38:11 +00:00
|
|
|
endif()
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
2023-05-02 16:42:16 +00:00
|
|
|
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
|
|
|
|
message(STATUS "PowerPC detected")
|
2023-11-17 16:11:23 +00:00
|
|
|
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le")
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mcpu=powerpc64le)
|
2023-11-17 16:11:23 +00:00
|
|
|
else()
|
2024-01-26 20:34:06 +00:00
|
|
|
list(APPEND ARCH_FLAGS -mcpu=native -mtune=native)
|
2023-11-17 16:11:23 +00:00
|
|
|
#TODO: Add targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
|
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
else()
|
|
|
|
message(STATUS "Unknown architecture")
|
2023-03-13 17:12:33 +00:00
|
|
|
endif()
|
|
|
|
|
2024-01-26 20:34:06 +00:00
|
|
|
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${ARCH_FLAGS}>")
|
|
|
|
add_compile_options("$<$<COMPILE_LANGUAGE:C>:${ARCH_FLAGS}>")
|
|
|
|
|
|
|
|
if (LLAMA_CUBLAS)
|
|
|
|
list(APPEND CUDA_CXX_FLAGS ${ARCH_FLAGS})
|
|
|
|
list(JOIN CUDA_CXX_FLAGS " " CUDA_CXX_FLAGS_JOINED) # pass host compiler flags as a single argument
|
|
|
|
if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL "")
|
|
|
|
list(APPEND CUDA_FLAGS -Xcompiler ${CUDA_CXX_FLAGS_JOINED})
|
|
|
|
endif()
|
|
|
|
add_compile_options("$<$<COMPILE_LANGUAGE:CUDA>:${CUDA_FLAGS}>")
|
|
|
|
endif()
|
|
|
|
|
2023-12-12 09:27:26 +00:00
|
|
|
if (MINGW)
|
|
|
|
# Target Windows 8 for PrefetchVirtualMemory
|
2024-01-14 08:41:44 +00:00
|
|
|
add_compile_definitions(_WIN32_WINNT=${LLAMA_WIN_VER})
|
2023-12-12 09:27:26 +00:00
|
|
|
endif()
|
|
|
|
|
2023-09-08 14:58:07 +00:00
|
|
|
#
|
|
|
|
# POSIX conformance
|
|
|
|
#
|
|
|
|
|
2023-09-08 12:09:21 +00:00
|
|
|
# clock_gettime came in POSIX.1b (1993)
|
|
|
|
# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
|
|
|
|
# posix_memalign came in POSIX.1-2001 / SUSv3
|
|
|
|
# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
|
|
|
|
add_compile_definitions(_XOPEN_SOURCE=600)
|
|
|
|
|
|
|
|
# Somehow in OpenBSD whenever POSIX conformance is specified
|
|
|
|
# some string functions rely on locale_t availability,
|
|
|
|
# which was introduced in POSIX.1-2008, forcing us to go higher
|
2023-09-08 14:58:07 +00:00
|
|
|
if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
|
2023-09-08 12:09:21 +00:00
|
|
|
remove_definitions(-D_XOPEN_SOURCE=600)
|
|
|
|
add_compile_definitions(_XOPEN_SOURCE=700)
|
2023-09-08 14:58:07 +00:00
|
|
|
endif()
|
2023-09-08 12:09:21 +00:00
|
|
|
|
|
|
|
# Data types, macros and functions related to controlling CPU affinity and
|
|
|
|
# some memory allocation are available on Linux through GNU extensions in libc
|
2023-09-08 14:58:07 +00:00
|
|
|
if (CMAKE_SYSTEM_NAME MATCHES "Linux")
|
2023-09-08 12:09:21 +00:00
|
|
|
add_compile_definitions(_GNU_SOURCE)
|
2023-09-08 14:58:07 +00:00
|
|
|
endif()
|
2023-09-08 12:09:21 +00:00
|
|
|
|
|
|
|
# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,
|
|
|
|
# and on macOS its availability depends on enabling Darwin extensions
|
|
|
|
# similarly on DragonFly, enabling BSD extensions is necessary
|
2023-09-11 11:49:06 +00:00
|
|
|
if (
|
|
|
|
CMAKE_SYSTEM_NAME MATCHES "Darwin" OR
|
|
|
|
CMAKE_SYSTEM_NAME MATCHES "iOS" OR
|
|
|
|
CMAKE_SYSTEM_NAME MATCHES "tvOS" OR
|
|
|
|
CMAKE_SYSTEM_NAME MATCHES "DragonFly"
|
|
|
|
)
|
2023-09-08 12:09:21 +00:00
|
|
|
add_compile_definitions(_DARWIN_C_SOURCE)
|
2023-09-08 14:58:07 +00:00
|
|
|
endif()
|
2023-09-08 12:09:21 +00:00
|
|
|
|
|
|
|
# alloca is a non-standard interface that is not visible on BSDs when
|
|
|
|
# POSIX conformance is specified, but not all of them provide a clean way
|
|
|
|
# to enable it in such cases
|
2023-09-08 14:58:07 +00:00
|
|
|
if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
2023-09-08 12:09:21 +00:00
|
|
|
add_compile_definitions(__BSD_VISIBLE)
|
2023-09-08 14:58:07 +00:00
|
|
|
endif()
|
|
|
|
if (CMAKE_SYSTEM_NAME MATCHES "NetBSD")
|
2023-09-08 12:09:21 +00:00
|
|
|
add_compile_definitions(_NETBSD_SOURCE)
|
2023-09-08 14:58:07 +00:00
|
|
|
endif()
|
|
|
|
if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
|
2023-09-08 12:09:21 +00:00
|
|
|
add_compile_definitions(_BSD_SOURCE)
|
2023-09-08 14:58:07 +00:00
|
|
|
endif()
|
2023-09-08 12:09:21 +00:00
|
|
|
|
2023-03-21 00:37:16 +00:00
|
|
|
#
|
2023-08-21 20:07:43 +00:00
|
|
|
# libraries
|
2023-03-21 00:37:16 +00:00
|
|
|
#
|
|
|
|
|
2023-08-21 20:07:43 +00:00
|
|
|
# ggml
|
|
|
|
|
2023-03-21 15:29:41 +00:00
|
|
|
add_library(ggml OBJECT
|
|
|
|
ggml.c
|
2023-04-20 01:14:14 +00:00
|
|
|
ggml.h
|
2023-07-30 13:58:01 +00:00
|
|
|
ggml-alloc.c
|
|
|
|
ggml-alloc.h
|
2023-10-08 17:19:14 +00:00
|
|
|
ggml-backend.c
|
|
|
|
ggml-backend.h
|
2023-10-29 16:32:28 +00:00
|
|
|
ggml-quants.c
|
|
|
|
ggml-quants.h
|
2024-01-29 20:50:50 +00:00
|
|
|
${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA}
|
|
|
|
${GGML_SOURCES_OPENCL} ${GGML_HEADERS_OPENCL}
|
|
|
|
${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL}
|
|
|
|
${GGML_SOURCES_MPI} ${GGML_HEADERS_MPI}
|
|
|
|
${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA}
|
|
|
|
${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL}
|
|
|
|
${GGML_SOURCES_KOMPUTE} ${GGML_HEADERS_KOMPUTE}
|
2024-02-16 17:05:56 +00:00
|
|
|
${GGML_SOURCES_VULKAN} ${GGML_HEADERS_VULKAN}
|
|
|
|
${GGML_SOURCES_ROCM} ${GGML_HEADERS_ROCM}
|
2023-06-04 20:34:30 +00:00
|
|
|
)
|
2023-03-21 15:29:41 +00:00
|
|
|
|
2023-06-15 17:51:26 +00:00
|
|
|
target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES})
|
2024-02-16 17:05:56 +00:00
|
|
|
target_compile_features (ggml PUBLIC c_std_11) # don't bump
|
|
|
|
|
2023-04-21 18:27:06 +00:00
|
|
|
target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
|
2023-04-22 13:31:56 +00:00
|
|
|
|
2023-06-17 07:49:42 +00:00
|
|
|
add_library(ggml_static STATIC $<TARGET_OBJECTS:ggml>)
|
2024-02-16 17:05:56 +00:00
|
|
|
|
2023-03-23 20:16:48 +00:00
|
|
|
if (BUILD_SHARED_LIBS)
|
|
|
|
set_target_properties(ggml PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
2023-06-17 07:49:42 +00:00
|
|
|
add_library(ggml_shared SHARED $<TARGET_OBJECTS:ggml>)
|
2023-06-19 15:10:37 +00:00
|
|
|
target_link_libraries(ggml_shared PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
|
2023-07-19 07:01:11 +00:00
|
|
|
install(TARGETS ggml_shared LIBRARY)
|
2023-03-23 20:16:48 +00:00
|
|
|
endif()
|
2023-03-22 05:32:36 +00:00
|
|
|
|
2023-08-21 20:07:43 +00:00
|
|
|
# llama
|
|
|
|
|
2023-03-22 16:37:10 +00:00
|
|
|
add_library(llama
|
2023-03-22 05:32:36 +00:00
|
|
|
llama.cpp
|
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
|
|
|
llama.h
|
2023-06-04 20:34:30 +00:00
|
|
|
)
|
2023-03-22 05:32:36 +00:00
|
|
|
|
|
|
|
target_include_directories(llama PUBLIC .)
|
2024-02-16 17:05:56 +00:00
|
|
|
target_compile_features (llama PUBLIC cxx_std_11) # don't bump
|
|
|
|
|
2023-06-04 20:34:30 +00:00
|
|
|
target_link_libraries(llama PRIVATE
|
|
|
|
ggml
|
|
|
|
${LLAMA_EXTRA_LIBS}
|
|
|
|
)
|
2023-04-22 13:31:56 +00:00
|
|
|
|
2023-03-23 20:16:48 +00:00
|
|
|
if (BUILD_SHARED_LIBS)
|
|
|
|
set_target_properties(llama PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
|
|
|
target_compile_definitions(llama PRIVATE LLAMA_SHARED LLAMA_BUILD)
|
2023-06-10 14:47:34 +00:00
|
|
|
if (LLAMA_METAL)
|
|
|
|
set_target_properties(llama PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal")
|
|
|
|
endif()
|
2023-03-23 20:16:48 +00:00
|
|
|
endif()
|
2023-03-21 00:37:16 +00:00
|
|
|
|
2023-09-14 17:04:40 +00:00
|
|
|
|
2023-08-21 20:07:43 +00:00
|
|
|
#
|
|
|
|
# install
|
|
|
|
#
|
|
|
|
|
2023-07-19 07:01:11 +00:00
|
|
|
include(GNUInstallDirs)
|
2023-09-14 17:04:40 +00:00
|
|
|
include(CMakePackageConfigHelpers)
|
|
|
|
|
|
|
|
set(LLAMA_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}
|
|
|
|
CACHE PATH "Location of header files")
|
|
|
|
set(LLAMA_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR}
|
|
|
|
CACHE PATH "Location of library files")
|
|
|
|
set(LLAMA_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR}
|
|
|
|
CACHE PATH "Location of binary files")
|
|
|
|
set(LLAMA_BUILD_NUMBER ${BUILD_NUMBER})
|
|
|
|
set(LLAMA_BUILD_COMMIT ${BUILD_COMMIT})
|
|
|
|
set(LLAMA_INSTALL_VERSION 0.0.${BUILD_NUMBER})
|
2023-10-02 09:51:49 +00:00
|
|
|
get_directory_property(LLAMA_TRANSIENT_DEFINES COMPILE_DEFINITIONS)
|
2023-09-14 17:04:40 +00:00
|
|
|
|
|
|
|
configure_package_config_file(
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/scripts/LlamaConfig.cmake.in
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/LlamaConfig.cmake
|
|
|
|
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Llama
|
|
|
|
PATH_VARS LLAMA_INCLUDE_INSTALL_DIR
|
|
|
|
LLAMA_LIB_INSTALL_DIR
|
|
|
|
LLAMA_BIN_INSTALL_DIR )
|
|
|
|
|
|
|
|
write_basic_package_version_file(
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/LlamaConfigVersion.cmake
|
|
|
|
VERSION ${LLAMA_INSTALL_VERSION}
|
|
|
|
COMPATIBILITY SameMajorVersion)
|
|
|
|
|
|
|
|
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/LlamaConfig.cmake
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/LlamaConfigVersion.cmake
|
|
|
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Llama)
|
|
|
|
|
2024-01-18 21:36:07 +00:00
|
|
|
set(GGML_PUBLIC_HEADERS "ggml.h" "ggml-alloc.h" "ggml-backend.h"
|
2024-02-16 17:05:56 +00:00
|
|
|
"${GGML_HEADERS_CUDA}" "${GGML_HEADERS_OPENCL}"
|
2023-09-14 17:04:40 +00:00
|
|
|
"${GGML_HEADERS_METAL}" "${GGML_HEADERS_MPI}" "${GGML_HEADERS_EXTRA}")
|
|
|
|
|
|
|
|
set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
|
|
|
|
install(TARGETS ggml PUBLIC_HEADER)
|
|
|
|
|
2023-09-15 08:07:40 +00:00
|
|
|
set_target_properties(llama PROPERTIES PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/llama.h)
|
2023-09-14 17:04:40 +00:00
|
|
|
install(TARGETS llama LIBRARY PUBLIC_HEADER)
|
|
|
|
|
2023-07-19 07:01:11 +00:00
|
|
|
install(
|
|
|
|
FILES convert.py
|
|
|
|
PERMISSIONS
|
|
|
|
OWNER_READ
|
|
|
|
OWNER_WRITE
|
|
|
|
OWNER_EXECUTE
|
|
|
|
GROUP_READ
|
|
|
|
GROUP_EXECUTE
|
|
|
|
WORLD_READ
|
|
|
|
WORLD_EXECUTE
|
|
|
|
DESTINATION ${CMAKE_INSTALL_BINDIR})
|
|
|
|
install(
|
|
|
|
FILES convert-lora-to-ggml.py
|
|
|
|
PERMISSIONS
|
|
|
|
OWNER_READ
|
|
|
|
OWNER_WRITE
|
|
|
|
OWNER_EXECUTE
|
|
|
|
GROUP_READ
|
|
|
|
GROUP_EXECUTE
|
|
|
|
WORLD_READ
|
|
|
|
WORLD_EXECUTE
|
|
|
|
DESTINATION ${CMAKE_INSTALL_BINDIR})
|
2023-08-16 20:09:49 +00:00
|
|
|
if (LLAMA_METAL)
|
|
|
|
install(
|
|
|
|
FILES ggml-metal.metal
|
|
|
|
PERMISSIONS
|
|
|
|
OWNER_READ
|
|
|
|
OWNER_WRITE
|
|
|
|
GROUP_READ
|
|
|
|
WORLD_READ
|
|
|
|
DESTINATION ${CMAKE_INSTALL_BINDIR})
|
|
|
|
endif()
|
2023-04-20 01:14:14 +00:00
|
|
|
|
2023-03-21 15:29:41 +00:00
|
|
|
#
|
|
|
|
# programs, examples and tests
|
|
|
|
#
|
|
|
|
|
2023-08-21 20:07:43 +00:00
|
|
|
add_subdirectory(common)
|
|
|
|
|
2023-03-21 15:29:41 +00:00
|
|
|
if (LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
|
2023-03-30 17:56:59 +00:00
|
|
|
include(CTest)
|
2023-03-21 15:29:41 +00:00
|
|
|
add_subdirectory(tests)
|
|
|
|
endif ()
|
|
|
|
|
2023-03-25 18:26:40 +00:00
|
|
|
if (LLAMA_BUILD_EXAMPLES)
|
|
|
|
add_subdirectory(examples)
|
2023-04-18 19:00:14 +00:00
|
|
|
add_subdirectory(pocs)
|
2023-03-25 18:26:40 +00:00
|
|
|
endif()
|