From 3d48f42efcd05381221654376e9f6f69d76af739 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Sun, 5 Nov 2023 04:40:08 -0800 Subject: [PATCH] llama : mark LLM_ARCH_STARCODER as full offload supported (#3945) as done in https://github.com/ggerganov/llama.cpp/pull/3827 --- llama.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/llama.cpp b/llama.cpp index cc0211ceb..e16539000 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5164,11 +5164,12 @@ static int llama_decode_internal( // If all tensors can be run on the GPU then using more than 1 thread is detrimental. const bool full_offload_supported = - model.arch == LLM_ARCH_LLAMA || - model.arch == LLM_ARCH_BAICHUAN || - model.arch == LLM_ARCH_FALCON || - model.arch == LLM_ARCH_REFACT || - model.arch == LLM_ARCH_MPT; + model.arch == LLM_ARCH_LLAMA || + model.arch == LLM_ARCH_BAICHUAN || + model.arch == LLM_ARCH_FALCON || + model.arch == LLM_ARCH_REFACT || + model.arch == LLM_ARCH_MPT || + model.arch == LLM_ARCH_STARCODER; const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 3; if (ggml_cpu_has_cublas() && full_offload_supported && fully_offloaded) {