root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-26 03:14:35 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

cuda : fix multi-gpu with tensor cores

Browse Source
This commit is contained in:
Georgi Gerganov 2023-10-27 23:11:50 +03:00
parent 706ff4c2e0
commit cd3e20fb50
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ggml-cuda.cu
View File

@ -461,7 +461,12 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA
#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128
#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
#ifdef GGML_CUDA_FORCE_MMQ
#define MUL_MAT_SRC1_COL_STRIDE 128
#else
// with tensor cores, we copy the entire hidden state to the devices in one go
#define MUL_MAT_SRC1_COL_STRIDE 4096
#endif
#define MAX_STREAMS 8
static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { nullptr };