2023-03-21 15:29:41 +00:00
cmake_minimum_required ( VERSION 3.12 ) # Don't bump this version for no reason
2023-03-21 00:37:16 +00:00
project ( "llama.cpp" C CXX )
2023-03-13 17:12:33 +00:00
2023-03-21 15:29:41 +00:00
set ( CMAKE_EXPORT_COMPILE_COMMANDS ON )
2023-03-13 19:22:15 +00:00
if ( NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE )
set ( CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE )
set_property ( CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo" )
endif ( )
2023-03-21 15:29:41 +00:00
set ( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${ CMAKE_BINARY_DIR } /bin )
if ( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
set ( LLAMA_STANDALONE ON )
# configure project version
# TODO
else ( )
set ( LLAMA_STANDALONE OFF )
endif ( )
if ( EMSCRIPTEN )
set ( BUILD_SHARED_LIBS_DEFAULT OFF )
option ( LLAMA_WASM_SINGLE_FILE "llama: embed WASM inside the generated llama.js" ON )
else ( )
if ( MINGW )
set ( BUILD_SHARED_LIBS_DEFAULT OFF )
else ( )
set ( BUILD_SHARED_LIBS_DEFAULT ON )
endif ( )
endif ( )
2023-03-21 00:37:16 +00:00
#
# Option list
#
2023-03-13 17:12:33 +00:00
2023-03-21 00:37:16 +00:00
# general
2023-05-25 21:07:29 +00:00
option ( LLAMA_STATIC "llama: static link libraries" OFF )
option ( LLAMA_NATIVE "llama: enable -march=native flag" OFF )
option ( LLAMA_LTO "llama: enable link time optimization" OFF )
2023-03-13 17:12:33 +00:00
2023-03-21 00:37:16 +00:00
# debug
2023-05-25 21:07:29 +00:00
option ( LLAMA_ALL_WARNINGS "llama: enable all compiler warnings" ON )
option ( LLAMA_ALL_WARNINGS_3RD_PARTY "llama: enable all compiler warnings in 3rd party libs" OFF )
option ( LLAMA_GPROF "llama: enable gprof" OFF )
2023-03-21 00:37:16 +00:00
# sanitizers
2023-05-25 21:07:29 +00:00
option ( LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" OFF )
option ( LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" OFF )
option ( LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF )
2023-03-21 00:37:16 +00:00
# instruction set specific
2023-05-25 21:07:29 +00:00
option ( LLAMA_AVX "llama: enable AVX" ON )
option ( LLAMA_AVX2 "llama: enable AVX2" ON )
option ( LLAMA_AVX512 "llama: enable AVX512" OFF )
option ( LLAMA_AVX512_VBMI "llama: enable AVX512-VBMI" OFF )
option ( LLAMA_AVX512_VNNI "llama: enable AVX512-VNNI" OFF )
option ( LLAMA_FMA "llama: enable FMA" ON )
2023-04-13 12:48:21 +00:00
# in MSVC F16C is implied with AVX2/AVX512
if ( NOT MSVC )
2023-05-25 21:07:29 +00:00
option ( LLAMA_F16C "llama: enable F16C" ON )
2023-04-13 12:48:21 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
# 3rd party libs
2023-06-04 20:34:30 +00:00
option ( LLAMA_ACCELERATE "llama: enable Accelerate framework" ON )
option ( LLAMA_BLAS "llama: use BLAS" OFF )
2023-05-27 14:24:06 +00:00
set ( LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor" )
2023-07-29 21:04:44 +00:00
option ( LLAMA_CUBLAS "llama: use CUDA" OFF )
2023-07-31 13:44:35 +00:00
#option(LLAMA_CUDA_CUBLAS "llama: use cuBLAS for prompt processing" OFF)
2023-07-05 12:19:42 +00:00
option ( LLAMA_CUDA_FORCE_DMMV "llama: use dmmv instead of mmvq CUDA kernels" OFF )
2023-06-04 20:34:30 +00:00
set ( LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels" )
2023-07-05 12:19:42 +00:00
set ( LLAMA_CUDA_MMV_Y "1" CACHE STRING "llama: y block size for mmv CUDA kernels" )
2023-07-31 17:52:22 +00:00
option ( LLAMA_CUDA_F16 "llama: use 16 bit floats for some calculations" OFF )
2023-06-16 17:08:44 +00:00
set ( LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K" )
2023-08-25 09:09:42 +00:00
option ( LLAMA_HIPBLAS "llama: use hipBLAS" OFF )
2023-06-04 20:34:30 +00:00
option ( LLAMA_CLBLAST "llama: use CLBlast" OFF )
option ( LLAMA_METAL "llama: use Metal" OFF )
2023-07-10 15:49:56 +00:00
option ( LLAMA_MPI "llama: use MPI" OFF )
2023-06-07 07:59:52 +00:00
option ( LLAMA_K_QUANTS "llama: use k-quants" ON )
k-quants : support for super-block size of 64 (#2001)
* k_quants: WIP super-blocks with 64 weights
* k_quants: WIP super-blocks with 64 weights
Q6_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q4_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q2_K scalar and AVX2 works. Q2_K is way too slow (it is actually slower
than the scalar implementation)
* k_quants: WIP super-blocks with 64 weights
Q3_K scalar and AVX2 works.
* k_quants: WIP super-blocks with 64 weights
Q5_K scalar and AVX2 works, and with that all
k_quants are done on AVX2 and scalar
* k_quants: WIP super-blocks with 64 weights
Q6_K working on CUDA. Cannot make it run quite as gast as
with super-blocks with 256 weigths: 8% slower on 4080,
20% slower on the 1660 (but there we fit 1 less layer on the
GPU because pf the larger model size), so some fraction of
these 20% is due to that,
* k_quants: WIP super-blocks with 64 weights
Q4_K working on CUDA. ~10% slower on GTX-1660,
16% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q2_K working on CUDA. ~3% slower on GTX-1660,
10% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q3_K working on CUDA.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on CUDA, and with this CUDA is done.
* k_quants: WIP super-blocks with 64 weights
Q6_K working on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Q4_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q2_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q3_K working on ARM_NEON, but quite a bit slower than 256 weights.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on ARM_NEON, but quite a bit slower than 256 weights.
With that, we have full support for ARM_NEON, although
performance is not quite there.
* k_quants: WIP super-blocks with 64 weights
Slightly more efficient Q3_K and Q5_K
* k_quants: WIP super-blocks with 64 weights
Another small improvement for Q3_K and Q5_K on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Yet another speedup for Q5_K on ARM_NEON.
We are now within 10% of the QK_K = 256 version.
* k_quants: WIP super-blocks with 64 weights
* We are able to pass preprocessor macros to the Metal
compiler
* Q6_K works and is actually slightly more efficient than
the QK_K = 256 version (25.2 ms vs 25.8 ms)
* k_quants: WIP super-blocks with 64 weights
Q4_K works on Metal and is actually slightly faster
than QK_K = 256 (21.95 ms vs 24.0 ms).
* k_quants: WIP super-blocks with 64 weights
Q2_K works on Metal and is very slightly faster
than QK_K = 256 (23.8 ms vs 24.2 ms).
* k_quants: WIP super-blocks with 64 weights
Q3_K works on Metal and is slightly faster
than QK_K = 256 (26.6 ms vs 28.3 ms).
* k_quants: WIP super-blocks with 64 weights
Q5_K works on Metal and is slightly faster
than QK_K = 256 (23.7 ms vs 26.3 ms).
* k_quants: call them _K, not _k, also on Metal
* k_quants: correctly define QK_K in llama.cpp
* Fixed bug in q4_K quantization added with the 64-block addition
* Simplify via lambda
* k_quants: swicth Q3_K to 4-bit scales when QK_K = 64
Otherwise there isn't much benefit from this
quantization type. There is some very slight loss
in accuracy, but we reduce size by ~7%.
E.g., for OpenLLaMA-3B, Q3_K_S perplexity is
8.6131 with 8-bit scales and 8.6352 with 4-bit,
while file size decreases from 1.53G to 1.44G.
* k_quants: switch Q4_K to 4-bit scales when QK_K = 64
Here the loss in accuracy is greater than for Q3_K,
but the Q4_K points still move further to the left on
the perplexity vs size curve.
* k_quants: forgot to add the Metal changes in last commit
* k_quants: change Q5_K to be type 0 when QK_K = 64
Still needs AVX2 implementation
* k_quants: AVX2 implementation for new 64-weight Q5_K
* k_quants: 10% faster ARM_NEON Q5_K dot product
* k_quants: fixed issue caused by merging with master
---------
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
2023-06-26 16:43:07 +00:00
option ( LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF )
2023-05-25 21:07:29 +00:00
option ( LLAMA_BUILD_TESTS "llama: build tests" ${ LLAMA_STANDALONE } )
option ( LLAMA_BUILD_EXAMPLES "llama: build examples" ${ LLAMA_STANDALONE } )
2023-07-04 14:05:27 +00:00
option ( LLAMA_BUILD_SERVER "llama: build server example" ON )
2023-03-21 15:29:41 +00:00
2023-05-01 16:23:47 +00:00
#
# Build info header
#
# Generate initial build-info.h
include ( ${ CMAKE_CURRENT_SOURCE_DIR } /scripts/build-info.cmake )
if ( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" )
2023-05-03 00:43:43 +00:00
set ( GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.git" )
# Is git submodule
if ( NOT IS_DIRECTORY "${GIT_DIR}" )
file ( READ ${ GIT_DIR } REAL_GIT_DIR_LINK )
string ( REGEX REPLACE "gitdir: (.*)\n$" "\\1" REAL_GIT_DIR ${ REAL_GIT_DIR_LINK } )
set ( GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${REAL_GIT_DIR}" )
endif ( )
2023-05-01 16:23:47 +00:00
# Add a custom target for build-info.h
add_custom_target ( BUILD_INFO ALL DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/build-info.h" )
# Add a custom command to rebuild build-info.h when .git/index changes
add_custom_command (
O U T P U T " $ { C M A K E _ C U R R E N T _ S O U R C E _ D I R } / b u i l d - i n f o . h "
C O M M E N T " G e n e r a t i n g b u i l d d e t a i l s f r o m G i t "
C O M M A N D $ { C M A K E _ C O M M A N D } - P " $ { C M A K E _ C U R R E N T _ S O U R C E _ D I R } / s c r i p t s / b u i l d - i n f o . c m a k e "
W O R K I N G _ D I R E C T O R Y $ { C M A K E _ C U R R E N T _ S O U R C E _ D I R }
2023-05-03 00:43:43 +00:00
D E P E N D S " $ { G I T _ D I R } / i n d e x "
2023-05-01 16:23:47 +00:00
V E R B A T I M
)
else ( )
message ( WARNING "Git repository not found; to enable automatic generation of build info, make sure Git is installed and the project is a Git repository." )
endif ( )
2023-03-21 00:37:16 +00:00
#
# Compile flags
#
2023-03-31 19:19:16 +00:00
set ( CMAKE_CXX_STANDARD 11 )
2023-03-21 00:37:16 +00:00
set ( CMAKE_CXX_STANDARD_REQUIRED true )
2023-03-31 19:19:16 +00:00
set ( CMAKE_C_STANDARD 11 )
2023-03-21 00:37:16 +00:00
set ( CMAKE_C_STANDARD_REQUIRED true )
set ( THREADS_PREFER_PTHREAD_FLAG ON )
find_package ( Threads REQUIRED )
2023-03-13 17:12:33 +00:00
if ( NOT MSVC )
if ( LLAMA_SANITIZE_THREAD )
2023-03-21 00:37:16 +00:00
add_compile_options ( -fsanitize=thread )
2023-03-25 21:38:11 +00:00
link_libraries ( -fsanitize=thread )
2023-03-13 17:12:33 +00:00
endif ( )
if ( LLAMA_SANITIZE_ADDRESS )
2023-03-21 00:37:16 +00:00
add_compile_options ( -fsanitize=address -fno-omit-frame-pointer )
2023-03-25 21:38:11 +00:00
link_libraries ( -fsanitize=address )
2023-03-13 17:12:33 +00:00
endif ( )
if ( LLAMA_SANITIZE_UNDEFINED )
2023-03-21 00:37:16 +00:00
add_compile_options ( -fsanitize=undefined )
2023-03-25 21:38:11 +00:00
link_libraries ( -fsanitize=undefined )
2023-03-13 17:12:33 +00:00
endif ( )
endif ( )
2023-03-21 00:37:16 +00:00
if ( APPLE AND LLAMA_ACCELERATE )
2023-03-13 17:12:33 +00:00
find_library ( ACCELERATE_FRAMEWORK Accelerate )
if ( ACCELERATE_FRAMEWORK )
message ( STATUS "Accelerate framework found" )
2023-03-21 00:37:16 +00:00
add_compile_definitions ( GGML_USE_ACCELERATE )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ${ ACCELERATE_FRAMEWORK } )
2023-03-13 17:12:33 +00:00
else ( )
message ( WARNING "Accelerate framework not found" )
endif ( )
endif ( )
2023-04-20 01:14:14 +00:00
2023-05-20 14:58:31 +00:00
if ( LLAMA_BLAS )
2023-03-21 00:37:16 +00:00
if ( LLAMA_STATIC )
set ( BLA_STATIC ON )
endif ( )
2023-05-20 14:58:31 +00:00
if ( $ ( CMAKE_VERSION ) VERSION_GREATER_EQUAL 3.22 )
set ( BLA_SIZEOF_INTEGER 8 )
endif ( )
2023-06-16 18:53:04 +00:00
2023-05-20 14:58:31 +00:00
set ( BLA_VENDOR ${ LLAMA_BLAS_VENDOR } )
2023-03-21 00:37:16 +00:00
find_package ( BLAS )
2023-06-16 18:53:04 +00:00
2023-03-21 00:37:16 +00:00
if ( BLAS_FOUND )
2023-05-20 14:58:31 +00:00
message ( STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}" )
2023-03-21 00:37:16 +00:00
2023-06-16 18:53:04 +00:00
if ( "${BLAS_INCLUDE_DIRS}" STREQUAL "" )
# BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
# see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
find_package ( PkgConfig REQUIRED )
if ( ${ LLAMA_BLAS_VENDOR } MATCHES "Generic" )
pkg_check_modules ( DepBLAS REQUIRED blas )
elseif ( ${ LLAMA_BLAS_VENDOR } MATCHES "OpenBLAS" )
pkg_check_modules ( DepBLAS REQUIRED openblas )
elseif ( ${ LLAMA_BLAS_VENDOR } MATCHES "FLAME" )
pkg_check_modules ( DepBLAS REQUIRED blis )
elseif ( ${ LLAMA_BLAS_VENDOR } MATCHES "ATLAS" )
pkg_check_modules ( DepBLAS REQUIRED blas-atlas )
elseif ( ${ LLAMA_BLAS_VENDOR } MATCHES "FlexiBLAS" )
pkg_check_modules ( DepBLAS REQUIRED flexiblas_api )
elseif ( ${ LLAMA_BLAS_VENDOR } MATCHES "Intel" )
# all Intel* libraries share the same include path
2023-07-21 10:26:34 +00:00
pkg_check_modules ( DepBLAS REQUIRED mkl-sdl )
2023-06-16 18:53:04 +00:00
elseif ( ${ LLAMA_BLAS_VENDOR } MATCHES "NVHPC" )
# this doesn't provide pkg-config
# suggest to assign BLAS_INCLUDE_DIRS on your own
if ( "${NVHPC_VERSION}" STREQUAL "" )
message ( WARNING "Better to set NVHPC_VERSION" )
else ( )
set ( DepBLAS_FOUND ON )
set ( DepBLAS_INCLUDE_DIRS "/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include" )
endif ( )
endif ( )
if ( DepBLAS_FOUND )
set ( BLAS_INCLUDE_DIRS ${ DepBLAS_INCLUDE_DIRS } )
else ( )
message ( WARNING "BLAS_INCLUDE_DIRS neither been provided nor been automatically"
" d e t e c t e d b y p k g c o n f i g , t r y i n g t o f i n d c b l a s . h f r o m p o s s i b l e p a t h s . . . " )
find_path ( BLAS_INCLUDE_DIRS
N A M E S c b l a s . h
H I N T S
/ u s r / i n c l u d e
/ u s r / l o c a l / i n c l u d e
/ u s r / i n c l u d e / o p e n b l a s
/ o p t / h o m e b r e w / o p t / o p e n b l a s / i n c l u d e
/ u s r / l o c a l / o p t / o p e n b l a s / i n c l u d e
/ u s r / i n c l u d e / x 8 6 _ 6 4 - l i n u x - g n u / o p e n b l a s / i n c l u d e
)
endif ( )
endif ( )
2023-06-15 17:51:26 +00:00
message ( STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}" )
2023-05-20 14:58:31 +00:00
add_compile_options ( ${ BLAS_LINKER_FLAGS } )
2023-03-21 00:37:16 +00:00
add_compile_definitions ( GGML_USE_OPENBLAS )
2023-07-09 08:12:20 +00:00
if ( ${ BLAS_INCLUDE_DIRS } MATCHES "mkl" AND ( ${ LLAMA_BLAS_VENDOR } MATCHES "Generic" OR ${ LLAMA_BLAS_VENDOR } MATCHES "Intel" ) )
add_compile_definitions ( GGML_BLAS_USE_MKL )
endif ( )
2023-05-20 14:58:31 +00:00
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ${ BLAS_LIBRARIES } )
2023-06-15 17:51:26 +00:00
set ( LLAMA_EXTRA_INCLUDES ${ LLAMA_EXTRA_INCLUDES } ${ BLAS_INCLUDE_DIRS } )
2023-05-20 14:58:31 +00:00
2023-03-21 00:37:16 +00:00
else ( )
2023-05-20 14:58:31 +00:00
message ( WARNING "BLAS not found, please refer to "
" h t t p s : / / c m a k e . o r g / c m a k e / h e l p / l a t e s t / m o d u l e / F i n d B L A S . h t m l #blas-lapack-vendors"
" t o s e t c o r r e c t L L A M A _ B L A S _ V E N D O R " )
2023-03-21 00:37:16 +00:00
endif ( )
endif ( )
2023-03-13 17:12:33 +00:00
k-quants : support for super-block size of 64 (#2001)
* k_quants: WIP super-blocks with 64 weights
* k_quants: WIP super-blocks with 64 weights
Q6_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q4_K scalar and AVX2 works
* k_quants: WIP super-blocks with 64 weights
Q2_K scalar and AVX2 works. Q2_K is way too slow (it is actually slower
than the scalar implementation)
* k_quants: WIP super-blocks with 64 weights
Q3_K scalar and AVX2 works.
* k_quants: WIP super-blocks with 64 weights
Q5_K scalar and AVX2 works, and with that all
k_quants are done on AVX2 and scalar
* k_quants: WIP super-blocks with 64 weights
Q6_K working on CUDA. Cannot make it run quite as gast as
with super-blocks with 256 weigths: 8% slower on 4080,
20% slower on the 1660 (but there we fit 1 less layer on the
GPU because pf the larger model size), so some fraction of
these 20% is due to that,
* k_quants: WIP super-blocks with 64 weights
Q4_K working on CUDA. ~10% slower on GTX-1660,
16% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q2_K working on CUDA. ~3% slower on GTX-1660,
10% slower on 4080.
* k_quants: WIP super-blocks with 64 weights
Q3_K working on CUDA.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on CUDA, and with this CUDA is done.
* k_quants: WIP super-blocks with 64 weights
Q6_K working on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Q4_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q2_K working on ARM_NEON, but quite a bit slower than 256 weights
* k_quants: WIP super-blocks with 64 weights
Q3_K working on ARM_NEON, but quite a bit slower than 256 weights.
* k_quants: WIP super-blocks with 64 weights
Q5_K working on ARM_NEON, but quite a bit slower than 256 weights.
With that, we have full support for ARM_NEON, although
performance is not quite there.
* k_quants: WIP super-blocks with 64 weights
Slightly more efficient Q3_K and Q5_K
* k_quants: WIP super-blocks with 64 weights
Another small improvement for Q3_K and Q5_K on ARM_NEON
* k_quants: WIP super-blocks with 64 weights
Yet another speedup for Q5_K on ARM_NEON.
We are now within 10% of the QK_K = 256 version.
* k_quants: WIP super-blocks with 64 weights
* We are able to pass preprocessor macros to the Metal
compiler
* Q6_K works and is actually slightly more efficient than
the QK_K = 256 version (25.2 ms vs 25.8 ms)
* k_quants: WIP super-blocks with 64 weights
Q4_K works on Metal and is actually slightly faster
than QK_K = 256 (21.95 ms vs 24.0 ms).
* k_quants: WIP super-blocks with 64 weights
Q2_K works on Metal and is very slightly faster
than QK_K = 256 (23.8 ms vs 24.2 ms).
* k_quants: WIP super-blocks with 64 weights
Q3_K works on Metal and is slightly faster
than QK_K = 256 (26.6 ms vs 28.3 ms).
* k_quants: WIP super-blocks with 64 weights
Q5_K works on Metal and is slightly faster
than QK_K = 256 (23.7 ms vs 26.3 ms).
* k_quants: call them _K, not _k, also on Metal
* k_quants: correctly define QK_K in llama.cpp
* Fixed bug in q4_K quantization added with the 64-block addition
* Simplify via lambda
* k_quants: swicth Q3_K to 4-bit scales when QK_K = 64
Otherwise there isn't much benefit from this
quantization type. There is some very slight loss
in accuracy, but we reduce size by ~7%.
E.g., for OpenLLaMA-3B, Q3_K_S perplexity is
8.6131 with 8-bit scales and 8.6352 with 4-bit,
while file size decreases from 1.53G to 1.44G.
* k_quants: switch Q4_K to 4-bit scales when QK_K = 64
Here the loss in accuracy is greater than for Q3_K,
but the Q4_K points still move further to the left on
the perplexity vs size curve.
* k_quants: forgot to add the Metal changes in last commit
* k_quants: change Q5_K to be type 0 when QK_K = 64
Still needs AVX2 implementation
* k_quants: AVX2 implementation for new 64-weight Q5_K
* k_quants: 10% faster ARM_NEON Q5_K dot product
* k_quants: fixed issue caused by merging with master
---------
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
2023-06-26 16:43:07 +00:00
if ( LLAMA_K_QUANTS )
set ( GGML_SOURCES_EXTRA ${ GGML_SOURCES_EXTRA } k_quants.c k_quants.h )
add_compile_definitions ( GGML_USE_K_QUANTS )
if ( LLAMA_QKK_64 )
add_compile_definitions ( GGML_QKK_64 )
endif ( )
endif ( )
2023-04-19 09:22:45 +00:00
if ( LLAMA_CUBLAS )
cmake_minimum_required ( VERSION 3.17 )
find_package ( CUDAToolkit )
if ( CUDAToolkit_FOUND )
message ( STATUS "cuBLAS found" )
2023-04-20 01:14:14 +00:00
enable_language ( CUDA )
2023-06-04 20:34:30 +00:00
set ( GGML_SOURCES_CUDA ggml-cuda.cu ggml-cuda.h )
2023-04-20 01:14:14 +00:00
2023-04-19 09:22:45 +00:00
add_compile_definitions ( GGML_USE_CUBLAS )
2023-07-31 13:44:35 +00:00
# if (LLAMA_CUDA_CUBLAS)
# add_compile_definitions(GGML_CUDA_CUBLAS)
# endif()
2023-07-05 12:19:42 +00:00
if ( LLAMA_CUDA_FORCE_DMMV )
add_compile_definitions ( GGML_CUDA_FORCE_DMMV )
endif ( )
2023-05-25 21:07:29 +00:00
add_compile_definitions ( GGML_CUDA_DMMV_X= ${ LLAMA_CUDA_DMMV_X } )
2023-07-05 12:19:42 +00:00
add_compile_definitions ( GGML_CUDA_MMV_Y= ${ LLAMA_CUDA_MMV_Y } )
if ( DEFINED LLAMA_CUDA_DMMV_Y )
add_compile_definitions ( GGML_CUDA_MMV_Y= ${ LLAMA_CUDA_DMMV_Y } ) # for backwards compatibility
endif ( )
2023-07-31 17:52:22 +00:00
if ( LLAMA_CUDA_F16 OR LLAMA_CUDA_DMMV_F16 )
add_compile_definitions ( GGML_CUDA_F16 )
2023-06-19 08:23:56 +00:00
endif ( )
2023-06-16 17:08:44 +00:00
add_compile_definitions ( K_QUANTS_PER_ITERATION= ${ LLAMA_CUDA_KQUANTS_ITER } )
2023-04-19 09:22:45 +00:00
if ( LLAMA_STATIC )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static )
else ( )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } CUDA::cudart CUDA::cublas CUDA::cublasLt )
endif ( )
2023-06-21 21:49:25 +00:00
if ( NOT DEFINED CMAKE_CUDA_ARCHITECTURES )
2023-07-31 13:44:35 +00:00
# 52 == lowest CUDA 12 standard
# 60 == f16 CUDA intrinsics
# 61 == integer CUDA intrinsics
2023-08-02 14:48:10 +00:00
# 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
if ( LLAMA_CUDA_F16 OR LLAMA_CUDA_DMMV_F16 )
2023-07-31 13:44:35 +00:00
set ( CMAKE_CUDA_ARCHITECTURES "60;61;70" ) # needed for f16 CUDA intrinsics
2023-06-21 21:49:25 +00:00
else ( )
2023-07-31 13:44:35 +00:00
set ( CMAKE_CUDA_ARCHITECTURES "52;61;70" ) # lowest CUDA 12 standard + lowest for integer intrinsics
2023-06-21 21:49:25 +00:00
endif ( )
endif ( )
message ( STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}" )
2023-04-19 09:22:45 +00:00
else ( )
message ( WARNING "cuBLAS not found" )
endif ( )
endif ( )
2023-06-04 20:34:30 +00:00
if ( LLAMA_METAL )
find_library ( FOUNDATION_LIBRARY Foundation REQUIRED )
find_library ( METAL_FRAMEWORK Metal REQUIRED )
find_library ( METALKIT_FRAMEWORK MetalKit REQUIRED )
set ( GGML_SOURCES_METAL ggml-metal.m ggml-metal.h )
add_compile_definitions ( GGML_USE_METAL )
2023-08-29 08:33:46 +00:00
#add_compile_definitions(GGML_METAL_NDEBUG)
2023-06-04 20:34:30 +00:00
# get full path to the file
#add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
# copy ggml-metal.metal to bin directory
configure_file ( ggml-metal.metal bin/ggml-metal.metal COPYONLY )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS }
$ { F O U N D A T I O N _ L I B R A R Y }
$ { M E T A L _ F R A M E W O R K }
$ { M E T A L K I T _ F R A M E W O R K }
)
endif ( )
2023-07-10 15:49:56 +00:00
if ( LLAMA_MPI )
cmake_minimum_required ( VERSION 3.10 )
find_package ( MPI )
if ( MPI_C_FOUND )
message ( STATUS "MPI found" )
set ( GGML_SOURCES_MPI ggml-mpi.c ggml-mpi.h )
add_compile_definitions ( GGML_USE_MPI )
add_compile_definitions ( ${ MPI_C_COMPILE_DEFINITIONS } )
set ( cxx_flags ${ cxx_flags } -Wno-cast-qual )
set ( c_flags ${ c_flags } -Wno-cast-qual )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ${ MPI_C_LIBRARIES } )
set ( LLAMA_EXTRA_INCLUDES ${ LLAMA_EXTRA_INCLUDES } ${ MPI_C_INCLUDE_DIRS } )
# Even if you're only using the C header, C++ programs may bring in MPI
# C++ functions, so more linkage is needed
if ( MPI_CXX_FOUND )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ${ MPI_CXX_LIBRARIES } )
endif ( )
else ( )
message ( WARNING "MPI not found" )
endif ( )
endif ( )
ggml : add CLBlast support (#1164)
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing
* Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers
* Finish merge of ClBlast support
* Move CLBlast implementation to separate file
Add buffer reuse code (adapted from slaren's cuda implementation)
* Add q4_2 and q4_3 CLBlast support, improve code
* Double CLBlast speed by disabling OpenBLAS thread workaround
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
* Fix device selection env variable names
* Fix cast in opencl kernels
* Add CLBlast to CMakeLists.txt
* Replace buffer pool with static buffers a, b, qb, c
Fix compile warnings
* Fix typos, use GGML_TYPE defines, improve code
* Improve btype dequant kernel selection code, add error if type is unsupported
* Improve code quality
* Move internal stuff out of header
* Use internal enums instead of CLBlast enums
* Remove leftover C++ includes and defines
* Make event use easier to read
Co-authored-by: Henri Vasserman <henv@hot.ee>
* Use c compiler for opencl files
* Simplify code, fix include
* First check error, then release event
* Make globals static, fix indentation
* Rename dequant kernels file to conform with other file names
* Fix import cl file name
---------
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-04-28 14:57:16 +00:00
if ( LLAMA_CLBLAST )
find_package ( CLBlast )
if ( CLBlast_FOUND )
message ( STATUS "CLBlast found" )
2023-06-04 20:34:30 +00:00
set ( GGML_SOURCES_OPENCL ggml-opencl.cpp ggml-opencl.h )
ggml : add CLBlast support (#1164)
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing
* Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers
* Finish merge of ClBlast support
* Move CLBlast implementation to separate file
Add buffer reuse code (adapted from slaren's cuda implementation)
* Add q4_2 and q4_3 CLBlast support, improve code
* Double CLBlast speed by disabling OpenBLAS thread workaround
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
* Fix device selection env variable names
* Fix cast in opencl kernels
* Add CLBlast to CMakeLists.txt
* Replace buffer pool with static buffers a, b, qb, c
Fix compile warnings
* Fix typos, use GGML_TYPE defines, improve code
* Improve btype dequant kernel selection code, add error if type is unsupported
* Improve code quality
* Move internal stuff out of header
* Use internal enums instead of CLBlast enums
* Remove leftover C++ includes and defines
* Make event use easier to read
Co-authored-by: Henri Vasserman <henv@hot.ee>
* Use c compiler for opencl files
* Simplify code, fix include
* First check error, then release event
* Make globals static, fix indentation
* Rename dequant kernels file to conform with other file names
* Fix import cl file name
---------
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <2141330+slaren@users.noreply.github.com>
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2023-04-28 14:57:16 +00:00
add_compile_definitions ( GGML_USE_CLBLAST )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } clblast )
else ( )
message ( WARNING "CLBlast not found" )
endif ( )
endif ( )
2023-08-25 09:09:42 +00:00
if ( LLAMA_HIPBLAS )
list ( APPEND CMAKE_PREFIX_PATH /opt/rocm )
if ( NOT ${ CMAKE_C_COMPILER_ID } MATCHES "Clang" )
message ( WARNING "Only LLVM is supported for HIP, hint: CC=/opt/rocm/llvm/bin/clang" )
endif ( )
if ( NOT ${ CMAKE_CXX_COMPILER_ID } MATCHES "Clang" )
message ( WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++" )
endif ( )
find_package ( hip )
find_package ( hipblas )
find_package ( rocblas )
if ( ${ hipblas_FOUND } AND ${ hip_FOUND } )
message ( STATUS "HIP and hipBLAS found" )
add_compile_definitions ( GGML_USE_HIPBLAS GGML_USE_CUBLAS )
add_library ( ggml-rocm OBJECT ggml-cuda.cu ggml-cuda.h )
if ( LLAMA_CUDA_FORCE_DMMV )
target_compile_definitions ( ggml-rocm PRIVATE GGML_CUDA_FORCE_DMMV )
endif ( )
target_compile_definitions ( ggml-rocm PRIVATE GGML_CUDA_DMMV_X= ${ LLAMA_CUDA_DMMV_X } )
target_compile_definitions ( ggml-rocm PRIVATE GGML_CUDA_MMV_Y= ${ LLAMA_CUDA_MMV_Y } )
target_compile_definitions ( ggml-rocm PRIVATE K_QUANTS_PER_ITERATION= ${ LLAMA_CUDA_KQUANTS_ITER } )
target_compile_definitions ( ggml-rocm PRIVATE CC_TURING=1000000000 )
set_source_files_properties ( ggml-cuda.cu PROPERTIES LANGUAGE CXX )
target_link_libraries ( ggml-rocm PRIVATE hip::device PUBLIC hip::host roc::rocblas roc::hipblas )
if ( LLAMA_STATIC )
message ( FATAL_ERROR "Static linking not supported for HIP/ROCm" )
endif ( )
set ( LLAMA_EXTRA_LIBS ${ LLAMA_EXTRA_LIBS } ggml-rocm )
else ( )
message ( WARNING "hipBLAS or HIP not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm" )
endif ( )
endif ( )
2023-03-13 17:12:33 +00:00
if ( LLAMA_ALL_WARNINGS )
if ( NOT MSVC )
2023-03-21 00:37:16 +00:00
set ( c_flags
- W a l l
- W e x t r a
- W p e d a n t i c
- W c a s t - q u a l
2023-03-28 16:48:20 +00:00
- W d o u b l e - p r o m o t i o n
- W s h a d o w
2023-03-21 00:37:16 +00:00
- W s t r i c t - p r o t o t y p e s
- W p o i n t e r - a r i t h
2023-07-26 18:00:04 +00:00
- W m i s s i n g - p r o t o t y p e s
2023-08-30 06:20:26 +00:00
- W e r r o r = i m p l i c i t - i n t
2023-09-01 13:34:50 +00:00
- W n o - u n u s e d - f u n c t i o n
2023-03-21 00:37:16 +00:00
)
set ( cxx_flags
- W a l l
- W e x t r a
- W p e d a n t i c
- W c a s t - q u a l
2023-03-28 16:51:55 +00:00
- W n o - u n u s e d - f u n c t i o n
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
- W n o - m u l t i c h a r
2023-03-21 00:37:16 +00:00
)
2023-09-01 13:34:50 +00:00
if ( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" )
# g++ only
set ( cxx_flags ${ cxx_flags } -Wno-format-truncation )
endif ( )
2023-03-13 17:12:33 +00:00
else ( )
# todo : msvc
endif ( )
2023-03-21 00:37:16 +00:00
add_compile_options (
" $ < $ < C O M P I L E _ L A N G U A G E : C > : $ { c _ f l a g s } > "
" $ < $ < C O M P I L E _ L A N G U A G E : C X X > : $ { c x x _ f l a g s } > "
)
2023-03-13 17:12:33 +00:00
endif ( )
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
if ( MSVC )
add_compile_definitions ( _CRT_SECURE_NO_WARNINGS )
2023-04-22 08:18:20 +00:00
if ( BUILD_SHARED_LIBS )
set ( CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON )
endif ( )
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_LTO )
include ( CheckIPOSupported )
check_ipo_supported ( RESULT result OUTPUT output )
if ( result )
set ( CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE )
else ( )
message ( WARNING "IPO is not supported: ${output}" )
endif ( )
endif ( )
# Architecture specific
# TODO: probably these flags need to be tweaked on some architectures
# feel free to update the Makefile for your architecture and send a pull request or issue
2023-03-13 17:12:33 +00:00
message ( STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}" )
2023-03-21 00:37:16 +00:00
if ( NOT MSVC )
if ( LLAMA_STATIC )
add_link_options ( -static )
if ( MINGW )
add_link_options ( -static-libgcc -static-libstdc++ )
endif ( )
endif ( )
if ( LLAMA_GPROF )
add_compile_options ( -pg )
endif ( )
if ( LLAMA_NATIVE )
add_compile_options ( -march=native )
endif ( )
endif ( )
2023-03-13 17:12:33 +00:00
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "arm" OR ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "aarch64" )
message ( STATUS "ARM detected" )
2023-03-21 00:37:16 +00:00
if ( MSVC )
# TODO: arm msvc?
else ( )
2023-04-30 18:48:38 +00:00
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "armv6" )
# Raspberry Pi 1, Zero
add_compile_options ( -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access )
endif ( )
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "armv7" )
# Raspberry Pi 2
add_compile_options ( -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations )
endif ( )
if ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "armv8" )
# Raspberry Pi 3, 4, Zero 2 (32-bit)
add_compile_options ( -mfp16-format=ieee -mno-unaligned-access )
endif ( )
2023-03-21 00:37:16 +00:00
endif ( )
elseif ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "^(x86_64|i686|AMD64)$" )
2023-03-13 17:12:33 +00:00
message ( STATUS "x86 detected" )
if ( MSVC )
2023-03-25 21:38:11 +00:00
if ( LLAMA_AVX512 )
2023-04-20 01:14:14 +00:00
add_compile_options ( $< $<COMPILE_LANGUAGE:C > :/arch:AVX512> )
add_compile_options ( $< $<COMPILE_LANGUAGE:CXX > :/arch:AVX512> )
2023-04-17 13:10:57 +00:00
# MSVC has no compile-time flags enabling specific
# AVX512 extensions, neither it defines the
# macros corresponding to the extensions.
# Do it manually.
if ( LLAMA_AVX512_VBMI )
2023-04-20 01:14:14 +00:00
add_compile_definitions ( $< $<COMPILE_LANGUAGE:C > :__AVX512VBMI__> )
add_compile_definitions ( $< $<COMPILE_LANGUAGE:CXX > :__AVX512VBMI__> )
2023-04-17 13:10:57 +00:00
endif ( )
if ( LLAMA_AVX512_VNNI )
2023-04-20 01:14:14 +00:00
add_compile_definitions ( $< $<COMPILE_LANGUAGE:C > :__AVX512VNNI__> )
add_compile_definitions ( $< $<COMPILE_LANGUAGE:CXX > :__AVX512VNNI__> )
2023-04-17 13:10:57 +00:00
endif ( )
2023-03-25 21:38:11 +00:00
elseif ( LLAMA_AVX2 )
2023-04-20 01:14:14 +00:00
add_compile_options ( $< $<COMPILE_LANGUAGE:C > :/arch:AVX2> )
add_compile_options ( $< $<COMPILE_LANGUAGE:CXX > :/arch:AVX2> )
2023-03-21 00:37:16 +00:00
elseif ( LLAMA_AVX )
2023-04-20 01:14:14 +00:00
add_compile_options ( $< $<COMPILE_LANGUAGE:C > :/arch:AVX> )
add_compile_options ( $< $<COMPILE_LANGUAGE:CXX > :/arch:AVX> )
2023-03-21 00:37:16 +00:00
endif ( )
2023-03-13 17:12:33 +00:00
else ( )
2023-04-13 12:48:21 +00:00
if ( LLAMA_F16C )
add_compile_options ( -mf16c )
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_FMA )
add_compile_options ( -mfma )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_AVX )
add_compile_options ( -mavx )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
if ( LLAMA_AVX2 )
add_compile_options ( -mavx2 )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-25 21:38:11 +00:00
if ( LLAMA_AVX512 )
add_compile_options ( -mavx512f )
2023-04-17 13:10:57 +00:00
add_compile_options ( -mavx512bw )
endif ( )
if ( LLAMA_AVX512_VBMI )
add_compile_options ( -mavx512vbmi )
endif ( )
if ( LLAMA_AVX512_VNNI )
add_compile_options ( -mavx512vnni )
2023-03-25 21:38:11 +00:00
endif ( )
2023-03-13 17:12:33 +00:00
endif ( )
2023-05-02 16:42:16 +00:00
elseif ( ${ CMAKE_SYSTEM_PROCESSOR } MATCHES "ppc64" )
message ( STATUS "PowerPC detected" )
add_compile_options ( -mcpu=native -mtune=native )
#TODO: Add targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
2023-03-21 00:37:16 +00:00
else ( )
message ( STATUS "Unknown architecture" )
2023-03-13 17:12:33 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
#
2023-08-21 20:07:43 +00:00
# libraries
2023-03-21 00:37:16 +00:00
#
2023-08-21 20:07:43 +00:00
# ggml
2023-03-21 15:29:41 +00:00
add_library ( ggml OBJECT
g g m l . c
2023-04-20 01:14:14 +00:00
g g m l . h
2023-07-30 13:58:01 +00:00
g g m l - a l l o c . c
g g m l - a l l o c . h
2023-06-04 20:34:30 +00:00
$ { G G M L _ S O U R C E S _ C U D A }
$ { G G M L _ S O U R C E S _ O P E N C L }
$ { G G M L _ S O U R C E S _ M E T A L }
2023-07-10 15:49:56 +00:00
$ { G G M L _ S O U R C E S _ M P I }
2023-06-07 07:59:52 +00:00
$ { G G M L _ S O U R C E S _ E X T R A }
2023-06-04 20:34:30 +00:00
)
2023-03-21 15:29:41 +00:00
2023-06-15 17:51:26 +00:00
target_include_directories ( ggml PUBLIC . ${ LLAMA_EXTRA_INCLUDES } )
2023-03-21 15:29:41 +00:00
target_compile_features ( ggml PUBLIC c_std_11 ) # don't bump
2023-04-21 18:27:06 +00:00
target_link_libraries ( ggml PUBLIC Threads::Threads ${ LLAMA_EXTRA_LIBS } )
2023-04-22 13:31:56 +00:00
2023-06-17 07:49:42 +00:00
add_library ( ggml_static STATIC $< TARGET_OBJECTS:ggml > )
2023-03-23 20:16:48 +00:00
if ( BUILD_SHARED_LIBS )
set_target_properties ( ggml PROPERTIES POSITION_INDEPENDENT_CODE ON )
2023-06-17 07:49:42 +00:00
add_library ( ggml_shared SHARED $< TARGET_OBJECTS:ggml > )
2023-06-19 15:10:37 +00:00
target_link_libraries ( ggml_shared PUBLIC Threads::Threads ${ LLAMA_EXTRA_LIBS } )
2023-07-19 07:01:11 +00:00
install ( TARGETS ggml_shared LIBRARY )
2023-03-23 20:16:48 +00:00
endif ( )
2023-03-22 05:32:36 +00:00
2023-08-21 20:07:43 +00:00
# llama
2023-03-22 16:37:10 +00:00
add_library ( llama
2023-03-22 05:32:36 +00:00
l l a m a . c p p
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 19:24:37 +00:00
l l a m a . h
2023-06-04 20:34:30 +00:00
)
2023-03-22 05:32:36 +00:00
target_include_directories ( llama PUBLIC . )
target_compile_features ( llama PUBLIC cxx_std_11 ) # don't bump
2023-06-04 20:34:30 +00:00
target_link_libraries ( llama PRIVATE
g g m l
$ { L L A M A _ E X T R A _ L I B S }
)
2023-04-22 13:31:56 +00:00
2023-03-23 20:16:48 +00:00
if ( BUILD_SHARED_LIBS )
set_target_properties ( llama PROPERTIES POSITION_INDEPENDENT_CODE ON )
target_compile_definitions ( llama PRIVATE LLAMA_SHARED LLAMA_BUILD )
2023-06-10 14:47:34 +00:00
if ( LLAMA_METAL )
set_target_properties ( llama PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal" )
endif ( )
2023-07-19 07:01:11 +00:00
install ( TARGETS llama LIBRARY )
2023-03-23 20:16:48 +00:00
endif ( )
2023-03-21 00:37:16 +00:00
2023-08-21 20:07:43 +00:00
#
# install
#
2023-07-19 07:01:11 +00:00
include ( GNUInstallDirs )
install (
F I L E S c o n v e r t . p y
P E R M I S S I O N S
O W N E R _ R E A D
O W N E R _ W R I T E
O W N E R _ E X E C U T E
G R O U P _ R E A D
G R O U P _ E X E C U T E
W O R L D _ R E A D
W O R L D _ E X E C U T E
D E S T I N A T I O N $ { C M A K E _ I N S T A L L _ B I N D I R } )
install (
F I L E S c o n v e r t - l o r a - t o - g g m l . p y
P E R M I S S I O N S
O W N E R _ R E A D
O W N E R _ W R I T E
O W N E R _ E X E C U T E
G R O U P _ R E A D
G R O U P _ E X E C U T E
W O R L D _ R E A D
W O R L D _ E X E C U T E
D E S T I N A T I O N $ { C M A K E _ I N S T A L L _ B I N D I R } )
2023-08-16 20:09:49 +00:00
if ( LLAMA_METAL )
install (
F I L E S g g m l - m e t a l . m e t a l
P E R M I S S I O N S
O W N E R _ R E A D
O W N E R _ W R I T E
G R O U P _ R E A D
W O R L D _ R E A D
D E S T I N A T I O N $ { C M A K E _ I N S T A L L _ B I N D I R } )
endif ( )
2023-04-20 01:14:14 +00:00
2023-03-21 15:29:41 +00:00
#
# programs, examples and tests
#
2023-08-21 20:07:43 +00:00
add_subdirectory ( common )
2023-03-21 15:29:41 +00:00
if ( LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION )
2023-03-30 17:56:59 +00:00
include ( CTest )
2023-03-21 15:29:41 +00:00
add_subdirectory ( tests )
endif ( )
2023-03-25 18:26:40 +00:00
if ( LLAMA_BUILD_EXAMPLES )
add_subdirectory ( examples )
2023-04-18 19:00:14 +00:00
add_subdirectory ( pocs )
2023-03-25 18:26:40 +00:00
endif ( )
