mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-12-24 10:24:35 +00:00
fix whitespace (#944)
This commit is contained in:
parent
ec29272175
commit
8cda5c981d
6
Makefile
6
Makefile
@ -171,15 +171,15 @@ embedding: examples/embedding/embedding.cpp ggml.o llama.o common.o
|
|||||||
|
|
||||||
libllama.so: llama.o ggml.o
|
libllama.so: llama.o ggml.o
|
||||||
$(CXX) $(CXXFLAGS) -shared -fPIC -o libllama.so llama.o ggml.o $(LDFLAGS)
|
$(CXX) $(CXXFLAGS) -shared -fPIC -o libllama.so llama.o ggml.o $(LDFLAGS)
|
||||||
|
|
||||||
#
|
#
|
||||||
# Tests
|
# Tests
|
||||||
#
|
#
|
||||||
|
|
||||||
benchmark: ggml.o
|
benchmark: ggml.o
|
||||||
$(CXX) $(CXXFLAGS) examples/benchmark/benchmark-q4_0-matmult.c ggml.o -o benchmark-q4_0-matmult $(LDFLAGS)
|
$(CXX) $(CXXFLAGS) examples/benchmark/benchmark-q4_0-matmult.c ggml.o -o benchmark-q4_0-matmult $(LDFLAGS)
|
||||||
./benchmark-q4_0-matmult
|
./benchmark-q4_0-matmult
|
||||||
|
|
||||||
.PHONY: tests
|
.PHONY: tests
|
||||||
tests:
|
tests:
|
||||||
bash ./tests/run-tests.sh
|
bash ./tests/run-tests.sh
|
||||||
|
@ -24,12 +24,12 @@
|
|||||||
|
|
||||||
float tensor_sum_elements(struct ggml_tensor * tensor) {
|
float tensor_sum_elements(struct ggml_tensor * tensor) {
|
||||||
float sum = 0;
|
float sum = 0;
|
||||||
if (tensor->type==6) {
|
if (tensor->type==6) {
|
||||||
for (int j = 0; j < tensor->ne[1]; j++) {
|
for (int j = 0; j < tensor->ne[1]; j++) {
|
||||||
for (int k = 0; k < tensor->ne[0]; k++) {
|
for (int k = 0; k < tensor->ne[0]; k++) {
|
||||||
sum += ((float *) tensor->data)[j*tensor->ne[0]+k];
|
sum += ((float *) tensor->data)[j*tensor->ne[0]+k];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return sum;
|
return sum;
|
||||||
}
|
}
|
||||||
@ -39,7 +39,7 @@ float tensor_sum_elements(struct ggml_tensor * tensor) {
|
|||||||
These are mapping to unknown
|
These are mapping to unknown
|
||||||
GGML_TYPE_I8,
|
GGML_TYPE_I8,
|
||||||
GGML_TYPE_I16,
|
GGML_TYPE_I16,
|
||||||
GGML_TYPE_I32,
|
GGML_TYPE_I32,
|
||||||
GGML_TYPE_COUNT,
|
GGML_TYPE_COUNT,
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@ -50,7 +50,7 @@ float tensor_sum_elements(struct ggml_tensor * tensor) {
|
|||||||
TENSOR->ne[0], TENSOR->ne[1], TENSOR->ne[2], TENSOR->nb[0], TENSOR->nb[1], TENSOR->nb[2]); \
|
TENSOR->ne[0], TENSOR->ne[1], TENSOR->ne[2], TENSOR->nb[0], TENSOR->nb[1], TENSOR->nb[2]); \
|
||||||
{ float sum = tensor_sum_elements(TENSOR); printf("Sum of tensor %s is %6.2f\n",#TENSOR, sum); }
|
{ float sum = tensor_sum_elements(TENSOR); printf("Sum of tensor %s is %6.2f\n",#TENSOR, sum); }
|
||||||
|
|
||||||
struct benchmark_params_struct {
|
struct benchmark_params_struct {
|
||||||
int32_t n_threads = 1;
|
int32_t n_threads = 1;
|
||||||
int32_t n_iterations = 10;
|
int32_t n_iterations = 10;
|
||||||
};
|
};
|
||||||
@ -67,7 +67,7 @@ void print_usage(int /*argc*/, char ** argv, struct benchmark_params_struct para
|
|||||||
|
|
||||||
int main(int argc, char ** argv) {
|
int main(int argc, char ** argv) {
|
||||||
|
|
||||||
|
|
||||||
struct benchmark_params_struct benchmark_params;
|
struct benchmark_params_struct benchmark_params;
|
||||||
|
|
||||||
bool invalid_param = false;
|
bool invalid_param = false;
|
||||||
@ -90,7 +90,7 @@ int main(int argc, char ** argv) {
|
|||||||
} else if (arg == "-h" || arg == "--help") {
|
} else if (arg == "-h" || arg == "--help") {
|
||||||
print_usage(argc, argv, benchmark_params);
|
print_usage(argc, argv, benchmark_params);
|
||||||
exit(0);
|
exit(0);
|
||||||
}
|
}
|
||||||
if (invalid_param) {
|
if (invalid_param) {
|
||||||
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
||||||
print_usage(argc, argv, benchmark_params);
|
print_usage(argc, argv, benchmark_params);
|
||||||
@ -101,9 +101,9 @@ int main(int argc, char ** argv) {
|
|||||||
|
|
||||||
// create the ggml context
|
// create the ggml context
|
||||||
printf("Starting Test\n");
|
printf("Starting Test\n");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
struct ggml_context * ctx;
|
struct ggml_context * ctx;
|
||||||
//const int sizex = 4096;
|
//const int sizex = 4096;
|
||||||
//const int sizey = 11008;
|
//const int sizey = 11008;
|
||||||
@ -111,31 +111,31 @@ int main(int argc, char ** argv) {
|
|||||||
#undef VERBOSE_DEBUGGING
|
#undef VERBOSE_DEBUGGING
|
||||||
#ifndef VERBOSE_DEBUGGING
|
#ifndef VERBOSE_DEBUGGING
|
||||||
const int sizey = 4096;
|
const int sizey = 4096;
|
||||||
const int sizex = 11008;
|
const int sizex = 11008;
|
||||||
const int sizez = 128;
|
const int sizez = 128;
|
||||||
#else
|
#else
|
||||||
/* Working - let's increase size */
|
/* Working - let's increase size */
|
||||||
const int sizey = 1;
|
const int sizey = 1;
|
||||||
const int sizex = (8*32);
|
const int sizex = (8*32);
|
||||||
const int sizez = 1;
|
const int sizez = 1;
|
||||||
|
|
||||||
/*const int sizey = 1;
|
/*const int sizey = 1;
|
||||||
const int sizex = 3*(8*32);
|
const int sizex = 3*(8*32);
|
||||||
const int sizez = 1;*/
|
const int sizez = 1;*/
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
//printf("Memsize required = %i\n", sizex*sizex);
|
//printf("Memsize required = %i\n", sizex*sizex);
|
||||||
ggml_type wtype = GGML_TYPE_F32;
|
ggml_type wtype = GGML_TYPE_F32;
|
||||||
|
|
||||||
size_t ctx_size = 0;
|
size_t ctx_size = 0;
|
||||||
ctx_size += sizex*sizey*ggml_type_sizef(wtype);
|
ctx_size += sizex*sizey*ggml_type_sizef(wtype);
|
||||||
ctx_size += sizex*sizey*ggml_type_sizef(wtype);
|
ctx_size += sizex*sizey*ggml_type_sizef(wtype);
|
||||||
ctx_size += sizex*sizey*ggml_type_sizef(GGML_TYPE_F32);
|
ctx_size += sizex*sizey*ggml_type_sizef(GGML_TYPE_F32);
|
||||||
ctx_size += sizex*sizeof(float);
|
ctx_size += sizex*sizeof(float);
|
||||||
ctx_size += 1024*1024*100;
|
ctx_size += 1024*1024*100;
|
||||||
|
|
||||||
printf("Allocating Memory of size %li byes, %li MB\n",ctx_size, (ctx_size/1024/1024));
|
printf("Allocating Memory of size %li byes, %li MB\n",ctx_size, (ctx_size/1024/1024));
|
||||||
|
|
||||||
struct ggml_init_params params = {
|
struct ggml_init_params params = {
|
||||||
/*.mem_size =*/ ctx_size,
|
/*.mem_size =*/ ctx_size,
|
||||||
/*.mem_buffer =*/ NULL,
|
/*.mem_buffer =*/ NULL,
|
||||||
@ -147,88 +147,88 @@ int main(int argc, char ** argv) {
|
|||||||
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
printf("Creating new tensors\n");
|
printf("Creating new tensors\n");
|
||||||
// printf("Creating new tensor m1\n");
|
// printf("Creating new tensor m1\n");
|
||||||
struct ggml_tensor * m11 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey);
|
struct ggml_tensor * m11 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey);
|
||||||
ggml_set_f32(m11, 1.0f);
|
ggml_set_f32(m11, 1.0f);
|
||||||
|
|
||||||
// printf("Creating new tensor m1\n");
|
// printf("Creating new tensor m1\n");
|
||||||
struct ggml_tensor * m12 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey);
|
struct ggml_tensor * m12 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey);
|
||||||
ggml_set_f32(m12, 1.5f);
|
ggml_set_f32(m12, 1.5f);
|
||||||
|
|
||||||
// printf("Creating new tensor m2\n");
|
// printf("Creating new tensor m2\n");
|
||||||
struct ggml_tensor * m2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizez);
|
struct ggml_tensor * m2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizez);
|
||||||
ggml_set_f32(m2, 2.0f);
|
ggml_set_f32(m2, 2.0f);
|
||||||
|
|
||||||
printf("\n------ Test 1 - Matrix Mult via F32 code ------------------------------------------------------------------------------\n");
|
printf("\n------ Test 1 - Matrix Mult via F32 code ------------------------------------------------------------------------------\n");
|
||||||
// printf("Creating new tensor m11xm2\n");
|
// printf("Creating new tensor m11xm2\n");
|
||||||
struct ggml_tensor * m11xm2 = ggml_mul_mat(ctx, m11, m2);
|
struct ggml_tensor * m11xm2 = ggml_mul_mat(ctx, m11, m2);
|
||||||
|
|
||||||
// printf("Creating compute graph\n");
|
// printf("Creating compute graph\n");
|
||||||
struct ggml_cgraph gf = ggml_build_forward(m11xm2);
|
struct ggml_cgraph gf = ggml_build_forward(m11xm2);
|
||||||
|
|
||||||
gf.n_threads=benchmark_params.n_threads;
|
gf.n_threads=benchmark_params.n_threads;
|
||||||
printf("cgraph->n_threads=%i\n",gf.n_threads);
|
printf("cgraph->n_threads=%i\n",gf.n_threads);
|
||||||
|
|
||||||
TENSOR_DUMP(m11);
|
TENSOR_DUMP(m11);
|
||||||
TENSOR_DUMP(m2);
|
TENSOR_DUMP(m2);
|
||||||
|
|
||||||
ggml_graph_compute(ctx, &gf);
|
ggml_graph_compute(ctx, &gf);
|
||||||
|
|
||||||
TENSOR_DUMP(gf.nodes[0]);
|
TENSOR_DUMP(gf.nodes[0]);
|
||||||
|
|
||||||
printf("\n------ Test 2 - Matrix Mult via Q4_0 code ------------------------------------------------------------------------------\n");
|
printf("\n------ Test 2 - Matrix Mult via Q4_0 code ------------------------------------------------------------------------------\n");
|
||||||
|
|
||||||
int32_t nelements = sizex*sizey;
|
int32_t nelements = sizex*sizey;
|
||||||
int32_t ne[2] = { sizex, sizey };
|
int32_t ne[2] = { sizex, sizey };
|
||||||
|
|
||||||
std::vector<int64_t> hist_cur(1 << 4, 0);
|
std::vector<int64_t> hist_cur(1 << 4, 0);
|
||||||
|
|
||||||
// Set up a the benchmark matrices
|
// Set up a the benchmark matrices
|
||||||
// printf("Creating new tensor q11 & Running quantize\n");
|
// printf("Creating new tensor q11 & Running quantize\n");
|
||||||
struct ggml_tensor * q11 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, sizex, sizey);
|
struct ggml_tensor * q11 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, sizex, sizey);
|
||||||
ggml_quantize_q4_0((const float *) m11->data, q11->data, nelements, ne[0], hist_cur.data());
|
ggml_quantize_q4_0((const float *) m11->data, q11->data, nelements, ne[0], hist_cur.data());
|
||||||
|
|
||||||
// Set up a the compute graph
|
// Set up a the compute graph
|
||||||
// printf("Creating new tensor q31\n");
|
// printf("Creating new tensor q31\n");
|
||||||
struct ggml_tensor * q31 = ggml_mul_mat(ctx, q11, m2);
|
struct ggml_tensor * q31 = ggml_mul_mat(ctx, q11, m2);
|
||||||
|
|
||||||
// printf("Creating compute graph\n");
|
// printf("Creating compute graph\n");
|
||||||
struct ggml_cgraph gf31 = ggml_build_forward(q31);
|
struct ggml_cgraph gf31 = ggml_build_forward(q31);
|
||||||
gf31.n_threads=benchmark_params.n_threads;
|
gf31.n_threads=benchmark_params.n_threads;
|
||||||
|
|
||||||
// Set up a second graph computation to make sure we override the CPU cache lines
|
// Set up a second graph computation to make sure we override the CPU cache lines
|
||||||
// printf("Creating new tensor q12 & Running quantize\n");
|
// printf("Creating new tensor q12 & Running quantize\n");
|
||||||
struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, sizex, sizey);
|
struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, sizex, sizey);
|
||||||
ggml_quantize_q4_0((const float *) m12->data, q12->data, nelements, ne[0], hist_cur.data());
|
ggml_quantize_q4_0((const float *) m12->data, q12->data, nelements, ne[0], hist_cur.data());
|
||||||
|
|
||||||
// printf("Creating new tensor q32\n");
|
// printf("Creating new tensor q32\n");
|
||||||
struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2);
|
struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2);
|
||||||
|
|
||||||
//printf("Creating compute graph\n");
|
//printf("Creating compute graph\n");
|
||||||
struct ggml_cgraph gf32 = ggml_build_forward(q32);
|
struct ggml_cgraph gf32 = ggml_build_forward(q32);
|
||||||
gf32.n_threads=benchmark_params.n_threads;
|
gf32.n_threads=benchmark_params.n_threads;
|
||||||
printf("cgraph->n_threads=%i\n",gf31.n_threads);
|
printf("cgraph->n_threads=%i\n",gf31.n_threads);
|
||||||
|
|
||||||
const int dimx = sizex;
|
const int dimx = sizex;
|
||||||
const int dimy = sizey;
|
const int dimy = sizey;
|
||||||
const int dimz = sizez;
|
const int dimz = sizez;
|
||||||
long long int flops_per_dot_product = dimy + dimy;
|
long long int flops_per_dot_product = dimy + dimy;
|
||||||
long long int flops_per_matrix = flops_per_dot_product * dimx * dimz; ;
|
long long int flops_per_matrix = flops_per_dot_product * dimx * dimz; ;
|
||||||
printf("Matrix Multiplication of (%i,%i,%i) x (%i,%i,%i) - aboout %6.2f gFLOPS\n\n", sizex, sizey, 1, sizex, sizez, 1, 1.0f*flops_per_matrix / 1000 / 1000 / 1000);
|
printf("Matrix Multiplication of (%i,%i,%i) x (%i,%i,%i) - aboout %6.2f gFLOPS\n\n", sizex, sizey, 1, sizex, sizez, 1, 1.0f*flops_per_matrix / 1000 / 1000 / 1000);
|
||||||
|
|
||||||
|
|
||||||
// Let's use the F32 result from above as a reference for the q4_0 multiplication
|
// Let's use the F32 result from above as a reference for the q4_0 multiplication
|
||||||
float sum_of_F32_reference = tensor_sum_elements(gf.nodes[0]);
|
float sum_of_F32_reference = tensor_sum_elements(gf.nodes[0]);
|
||||||
|
|
||||||
|
|
||||||
printf("Iteration;NThreads; SizeX; SizeY; SizeZ; Required_FLOPS; Elapsed_u_Seconds; FLOPS_per_u_Second\n");
|
printf("Iteration;NThreads; SizeX; SizeY; SizeZ; Required_FLOPS; Elapsed_u_Seconds; FLOPS_per_u_Second\n");
|
||||||
printf("==============================================================================================\n");
|
printf("==============================================================================================\n");
|
||||||
|
|
||||||
for (int i=0;i<benchmark_params.n_iterations ;i++) {
|
for (int i=0;i<benchmark_params.n_iterations ;i++) {
|
||||||
|
|
||||||
long long int start = ggml_time_us();
|
long long int start = ggml_time_us();
|
||||||
//printf("Running ggml_graph_compute\n");
|
//printf("Running ggml_graph_compute\n");
|
||||||
ggml_graph_compute(ctx, &gf31);
|
ggml_graph_compute(ctx, &gf31);
|
||||||
@ -238,15 +238,15 @@ int main(int argc, char ** argv) {
|
|||||||
float flops_per_usec = (1.0f*flops_per_matrix)/usec;
|
float flops_per_usec = (1.0f*flops_per_matrix)/usec;
|
||||||
printf("%9i;%8i;%6i;%6i;%6i;%15lli;%18lli;%19.2f\n",
|
printf("%9i;%8i;%6i;%6i;%6i;%15lli;%18lli;%19.2f\n",
|
||||||
i,
|
i,
|
||||||
gf31.n_threads,
|
gf31.n_threads,
|
||||||
sizex, sizey, sizez, flops_per_matrix,
|
sizex, sizey, sizez, flops_per_matrix,
|
||||||
usec,flops_per_usec);
|
usec,flops_per_usec);
|
||||||
|
|
||||||
#ifdef VERBOSE_DEBUGGING
|
#ifdef VERBOSE_DEBUGGING
|
||||||
TENSOR_DUMP("res",gf31.nodes[0])
|
TENSOR_DUMP("res",gf31.nodes[0])
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// Check that the matrix multiplication result is in the right ballpark
|
// Check that the matrix multiplication result is in the right ballpark
|
||||||
// We cannot use the exact value from the F32 multiplication because the quantizuation will be slightly different
|
// We cannot use the exact value from the F32 multiplication because the quantizuation will be slightly different
|
||||||
float sum_of_Q4_result = tensor_sum_elements(gf31.nodes[0]);
|
float sum_of_Q4_result = tensor_sum_elements(gf31.nodes[0]);
|
||||||
float delta = abs(sum_of_Q4_result - sum_of_F32_reference);
|
float delta = abs(sum_of_Q4_result - sum_of_F32_reference);
|
||||||
@ -254,17 +254,17 @@ int main(int argc, char ** argv) {
|
|||||||
|
|
||||||
if (delta > allowed_delta) {
|
if (delta > allowed_delta) {
|
||||||
printf("\nABORT - ERROR in Matrix Multiplication result - expected %6.2f, got %6.2f (delta %6.2f > allowed_delta %6.2f)\n",
|
printf("\nABORT - ERROR in Matrix Multiplication result - expected %6.2f, got %6.2f (delta %6.2f > allowed_delta %6.2f)\n",
|
||||||
sum_of_F32_reference,
|
sum_of_F32_reference,
|
||||||
sum_of_Q4_result,
|
sum_of_Q4_result,
|
||||||
delta,
|
delta,
|
||||||
allowed_delta
|
allowed_delta
|
||||||
);
|
);
|
||||||
exit(0);
|
exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Running a different graph computation to make sure we override the CPU cache lines
|
// Running a different graph computation to make sure we override the CPU cache lines
|
||||||
ggml_graph_compute(ctx, &gf32);
|
ggml_graph_compute(ctx, &gf32);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user