diff --git a/ggml.c b/ggml.c index a0be068d6..fd3a51ba7 100644 --- a/ggml.c +++ b/ggml.c @@ -6977,7 +6977,6 @@ static struct ggml_tensor * ggml_rope_impl( float xpos_base, bool xpos_down, bool inplace) { - GGML_ASSERT(n_past >= 0); bool is_node = false; if (a->grad) { @@ -12645,8 +12644,6 @@ static void ggml_compute_forward_rope_f32( memcpy(&xpos_base, (int32_t *) dst->op_params + 6, sizeof(float)); memcpy(&xpos_down, (int32_t *) dst->op_params + 7, sizeof(bool)); - assert(n_past >= 0); - GGML_TENSOR_UNARY_OP_LOCALS; //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3); @@ -12674,12 +12671,14 @@ static void ggml_compute_forward_rope_f32( const float theta_scale = powf(freq_base, -2.0f/n_dims); + const bool is_skip = mode & 1; const bool is_neox = mode & 2; const bool is_glm = mode & 4; + const bool is_diff = mode & 8; // TODO: temporary for (int64_t i3 = 0; i3 < ne3; i3++) { - for (int64_t i2 = ((mode & 1) == 0 ? 0 : n_past); i2 < ne2; i2++) { - const int64_t p = ((mode & 1) == 0 ? n_past + i2 : i2); + for (int64_t i2 = (is_skip ? n_past : 0); i2 < ne2; i2++) { + const int64_t p = is_diff ? n_past : is_skip ? i2 : n_past + i2; for (int64_t i1 = 0; i1 < ne1; i1++) { if (ir++ < ir0) continue; if (ir > ir1) break; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 916dc9d05..a19e1376e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -37,6 +37,8 @@ llama_build_and_test_executable(test-llama-grammar.cpp) llama_build_and_test_executable(test-grad0.cpp) # SLOW # llama_build_and_test_executable(test-opt.cpp) # SLOW +llama_build_and_test_executable(test-rope.cpp) + # dummy executable - not installed get_filename_component(TEST_TARGET test-c.c NAME_WE) add_executable(${TEST_TARGET} test-c.c) diff --git a/tests/test-rope.cpp b/tests/test-rope.cpp new file mode 100644 index 000000000..a35bbd353 --- /dev/null +++ b/tests/test-rope.cpp @@ -0,0 +1,211 @@ +#include "ggml.h" + +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wdouble-promotion" +#endif + +#define MAX_NARGS 3 + +#undef MIN +#undef MAX +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + +#define GGML_SILU_FP16 + +// +// logging +// + +#if (GGML_DEBUG >= 1) +#define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG(...) +#endif + +#if (GGML_DEBUG >= 5) +#define GGML_PRINT_DEBUG_5(...) printf(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG_5(...) +#endif + +#if (GGML_DEBUG >= 10) +#define GGML_PRINT_DEBUG_10(...) printf(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG_10(...) +#endif + +#define GGML_PRINT(...) printf(__VA_ARGS__) + +static float frand(void) { + return (float)rand()/(float)RAND_MAX; +} + +static int irand(int n) { + if (n == 0) return 0; + return rand()%n; +} + +static void get_random_dims(int64_t * dims, int ndims) { + dims[0] = dims[1] = dims[2] = dims[3] = 1; + + for (int i = 0; i < ndims; i++) { + dims[i] = 1 + irand(4); + } +} + +static struct ggml_tensor * get_random_tensor_f32( + struct ggml_context * ctx0, + int ndims, + const int64_t ne[], + float fmin, + float fmax) { + struct ggml_tensor * result = ggml_new_tensor(ctx0, GGML_TYPE_F32, ndims, ne); + + switch (ndims) { + case 1: + for (int i0 = 0; i0 < ne[0]; i0++) { + ((float *)result->data)[i0] = frand()*(fmax - fmin) + fmin; + } + break; + case 2: + for (int i1 = 0; i1 < ne[1]; i1++) { + for (int i0 = 0; i0 < ne[0]; i0++) { + ((float *)result->data)[i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin; + } + } + break; + case 3: + for (int i2 = 0; i2 < ne[2]; i2++) { + for (int i1 = 0; i1 < ne[1]; i1++) { + for (int i0 = 0; i0 < ne[0]; i0++) { + ((float *)result->data)[i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin; + } + } + } + break; + case 4: + for (int i3 = 0; i3 < ne[3]; i3++) { + for (int i2 = 0; i2 < ne[2]; i2++) { + for (int i1 = 0; i1 < ne[1]; i1++) { + for (int i0 = 0; i0 < ne[0]; i0++) { + ((float *)result->data)[i3*ne[2]*ne[1]*ne[0] + i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin; + } + } + } + } + break; + default: + assert(false); + }; + + return result; +} + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +int main(int /*argc*/, const char ** /*argv*/) { + struct ggml_init_params params = { + /* .mem_size = */ 128*1024*1024, + /* .mem_buffer = */ NULL, + /* .no_alloc = */ false, + }; + + std::vector work_buffer; + + struct ggml_context * ctx0 = ggml_init(params); + + struct ggml_tensor * x; + + // rope f32 + for (int m = 0; m < 3; ++m) { + const int ndims = 4; + + const int64_t n_rot = 128; + const int64_t ne[4] = { 2*n_rot, 32, 73, 1 }; + + const int n_past_0 = 100; + const int n_past_1 = 33; + + // test mode 0, 2, 4 (standard, GPT-NeoX, GLM) + const int mode = m == 0 ? 0 : m == 1 ? 2 : 4; + + x = get_random_tensor_f32(ctx0, ndims, ne, -1.0f, 1.0f); + + // 100, 101, 102, ..., 172 + struct ggml_tensor * r0 = ggml_rope(ctx0, x, n_past_0, n_rot, mode, 1024); + // -67, -67, -67, ..., -67 + struct ggml_tensor * r1 = ggml_rope(ctx0, r0, n_past_1 - n_past_0, n_rot, mode + 8, 1024); // diff mode + + // 33, 34, 35, ..., 105 + struct ggml_tensor * r2 = ggml_rope(ctx0, x, n_past_1, n_rot, mode, 1024); + + ggml_cgraph * gf = ggml_new_graph(ctx0); + + ggml_build_forward_expand(gf, r0); + ggml_build_forward_expand(gf, r1); + ggml_build_forward_expand(gf, r2); + + ggml_graph_compute_helper(work_buffer, gf, 4); + + // check that r1 and r2 are the same + { + double sum0 = 0.0f; + double sum1 = 0.0f; + double diff = 0.0f; + + const float * r1_data = (float *) r1->data; + const float * r2_data = (float *) r2->data; + + const int n_elements = ggml_nelements(r1); + + for (int i = 0; i < n_elements; ++i) { + sum0 += fabs(r1_data[i]); + sum1 += fabs(r2_data[i]); + diff += fabs(r1_data[i] - r2_data[i]); + //if (fabs(r1_data[i] - r2_data[i]) > 0.0001f) { + // printf("%d: %f %f\n", i, r1_data[i], r2_data[i]); + // printf("diff: %f\n", fabs(r1_data[i] - r2_data[i])); + //} + } + + //for (int i = 4096; i < 4096 + 128; ++i) { + // printf("%f %f\n", r1_data[i], r2_data[i]); + //} + + printf("mode: %d\n", mode); + printf("sum0: %f\n", sum0); + printf("sum1: %f\n", sum1); + printf("diff: %f\n", diff); + printf("rel err: %f\n", diff / sum0); + printf("rel err: %f\n", diff / sum1); + + GGML_ASSERT(diff / sum0 < 0.0001f); + GGML_ASSERT(diff / sum1 < 0.0001f); + } + } + + ggml_free(ctx0); + + return 0; +} +