root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-05 00:04:36 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Support for gguf.

Browse Source
This commit is contained in:
Adam Treat 2023-09-21 12:39:33 -04:00 committed by cebtenzzre
parent 2c24d67e7b
commit 1b1416d7b7
1 changed files with 44 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
llama.cpp
View File

@ -514,6 +514,9 @@ static std::string llama_format_win_err(DWORD err) {
struct llama_buffer { struct llama_buffer {
void * data = NULL; void * data = NULL;
size_t size = 0; size_t size = 0;
#if defined(GGML_USE_KOMPUTE)
ggml_vk_memory memory;
#endif
// fallback to malloc / free // fallback to malloc / free
// useful in cases where CUDA can try to allocate PINNED memory // useful in cases where CUDA can try to allocate PINNED memory
@ -522,6 +525,14 @@ struct llama_buffer {
void resize(size_t n) { void resize(size_t n) {
llama_host_free(data); llama_host_free(data);
#if defined(GGML_USE_KOMPUTE)
if (ggml_vk_has_device()) {
this->memory = ggml_vk_allocate(n);
this->data = (uint8_t*)memory.data;
this->size = n;
return;
}
#endif
data = llama_host_malloc(n); data = llama_host_malloc(n);
if (!data) { if (!data) {
fallback = true; fallback = true;
@ -536,6 +547,13 @@ struct llama_buffer {
~llama_buffer() { ~llama_buffer() {
if (data) { if (data) {
#if defined(GGML_USE_KOMPUTE)
if (ggml_vk_has_device()) {
ggml_vk_free_memory(memory);
data = NULL;
return;
}
#endif
if (fallback) { // NOLINT if (fallback) { // NOLINT
free(data); free(data);
} else { } else {
@ -1398,6 +1416,9 @@ struct llama_model_loader {
use_mmap = false; use_mmap = false;
} }
#if defined(GGML_USE_KOMPUTE)
use_mmap = false;
#endif
this->use_mmap = use_mmap; this->use_mmap = use_mmap;
} }
@ -6470,6 +6491,23 @@ struct llama_context * llama_new_context_with_model(
LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "alloc", ctx->buf_alloc.data, ctx->buf_alloc.size, 0)); LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "alloc", ctx->buf_alloc.data, ctx->buf_alloc.size, 0));
#undef LLAMA_METAL_CHECK_BUF #undef LLAMA_METAL_CHECK_BUF
} }
#elif defined(GGML_USE_KOMPUTE)
if (ggml_vk_has_device() && params.n_gpu_layers > 0
&& (model->ftype == LLAMA_FTYPE_ALL_F32
|| model->ftype == LLAMA_FTYPE_MOSTLY_F16
|| model->ftype == LLAMA_FTYPE_MOSTLY_Q4_0)) {
// this allocates all Vulkan resources and memory buffers
ctx->ctx_kompute = ggml_vk_init();
const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx);
printf("%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0);
ggml_vk_add_buffer(ctx->ctx_kompute, "data", ctx->model.buf.memory);
ggml_vk_add_buffer(ctx->ctx_kompute, "eval", ctx->buf_compute.memory);
ggml_vk_add_buffer(ctx->ctx_kompute, "kv", ctx->kv_self.buf.memory);
ggml_vk_add_buffer(ctx->ctx_kompute, "alloc", ctx->buf_alloc.memory);
}
#endif #endif
} }
@ -6503,7 +6541,13 @@ static struct llama_context * llama_init_from_file(
} }
void llama_free(struct llama_context * ctx) { void llama_free(struct llama_context * ctx) {
#ifdef GGML_USE_KOMPUTE
ggml_vk_free(ctx->ctx_kompute);
#endif
delete ctx; delete ctx;
#ifdef GGML_USE_KOMPUTE
ggml_vk_free_device();
#endif
} }
int llama_n_vocab(const struct llama_context * ctx) { int llama_n_vocab(const struct llama_context * ctx) {